{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "089abfb7",
   "metadata": {},
   "source": [
    "# Building Blocks of PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8fd7272",
   "metadata": {},
   "source": [
    "#### System Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dd05ac7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18be7474",
   "metadata": {},
   "source": [
    "#### PyTorch Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e8fad9a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torchvision.io import read_image\n",
    "from torch.utils.data import Dataset\n",
    "from torchvision import transforms as T\n",
    "from torch.nn import functional as F \n",
    "from torch import optim\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0e963af",
   "metadata": {},
   "source": [
    "#### Loading MNIST Dataset using Custom Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fcb7dabc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomMNISTDataset(Dataset):\n",
    "\tdef __init__(self, dataset_path):\n",
    "\t\tself.dataset_path = dataset_path\n",
    "\t\tself.imgs, self.labels = [], []\n",
    "\t\tfor dirpath, _, filenames in os.walk(self.dataset_path):\n",
    "\t\t\tself.imgs.extend([f\"{dirpath}/{i}\" for i in filenames])\n",
    "\t\t\tself.labels.extend([f\"{dirpath.split('/')[-1]}\" for _ in filenames])\n",
    "\t\t\tself.transform = T.Compose([T.Lambda(torch.flatten)])\n",
    "\t\t\tself.target_transform = T.Compose([T.Lambda(lambda x: torch.LongTensor([x])), T.Lambda(lambda x: F.one_hot(x, 10))])\n",
    "\n",
    "\tdef __len__(self):\n",
    "\t\t# Returns the Size of the Dataset\n",
    "\t\treturn len(self.imgs)\n",
    "\n",
    "\tdef __getitem__(self, idx):\n",
    "\t\t# Returns a Sample from the data\n",
    "\t\timage = read_image(self.imgs[idx])\n",
    "\t\tlabel = int(self.labels[idx]) # Converting String Label to Integer\n",
    "\t\timage = self.transform(image)\n",
    "\t\timage = image / 255.0\n",
    "\t\tlabel = self.target_transform(label)\n",
    "\t\treturn image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "efcd3d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataloader = DataLoader(CustomMNISTDataset('/Users/atifadib/Deep_Learning_with_PyTorch/chapter3/output/'),\n",
    "                              batch_size=64, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8df7af0c",
   "metadata": {},
   "source": [
    "#### Single Sample from MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3873841a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAGzCAYAAAAogL7TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8EklEQVR4nO3deXgUVf7+/bsTkk6ALASyEJYAYRRl04kQwAUETEBEUBZRGQMygBpQYFDEHwroKIKOGyIqjqAMuKAorvBFBNxYRhSRQREiyBoElCQESEJynj940tIkkVQDaXJ4v66rrytdXZ/Tp6urc3dVddVxGWOMAACAtQL83QEAAHBmEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAy51TYe9yuTRhwgR/dwMnGDBggKpXr35a2+zQoYM6dOhwWts8V7z55puKiorSwYMHJUlbt26Vy+Xy3N566y3PvAMGDPBMb9asmb+6DFQ6a9euLfNz1bNnz1I/Vxs2bFCVKlW0fv16x8/nOOy///579e7dWwkJCQoJCVGdOnV01VVXaerUqY6fvLJr0KCBrrnmGn93wy/O5dcuHfvQTZgwQVu3bvV3V06rwsJCjR8/XsOHDy/xBWzIkCGaPXu2Wrdu7TW9Vq1amj17th599NES7X311Ve67LLLVLVqVcXFxenOO+/0fInw1Q8//KAuXbqoevXqioqK0t/+9jft3bv3lNrcuXOn+vbtq8jISIWHh6tHjx76+eefT6nNAwcOaMiQIYqOjla1atV05ZVX6ptvvjmlNvPy8jRmzBjFx8crNDRUycnJWrx48Sm1WVRUpClTpqhhw4YKCQlRixYt9Nprr51Sm9OnT1efPn1Uv359uVwuDRgw4JTaK2bT+pSQkKDZs2frvvvuK1E/cuRIzZ49W02aNPGafuGFF6pbt2564IEHnHfKOPDll1+a4OBg07hxY/PQQw+ZGTNmmAceeMCkpKSYxMREJ035hSQzfvz409ZeQkKC6dat22lrrzI5na89LS3NVKtW7bS0Vax9+/amffv2p7XN482bN89IMkuXLj1jz+EP77zzjnG5XGbHjh2eaVu2bDGSzMyZM0vMn5aWZhISEkpt69tvvzUhISHm4osvNtOnTzf/7//9P+N2u02XLl187t/27dtNrVq1TGJionn66afNww8/bGrUqGFatmxp8vLyfGozJyfH/OUvfzExMTFm8uTJ5oknnjD16tUzdevWNfv27fOpzcLCQtOuXTtTrVo1M2HCBPPss8+aCy+80ISFhZmffvrJpzaNMaZfv36mSpUqZvTo0eaFF14wbdu2NVWqVDGff/65z23ee++9RpIZPHiwefHFF023bt2MJPPaa6/53GZCQoKJiooyXbp0MVWqVDFpaWk+t1XM1vVp6dKlRpKZN29eicfat29vmjZt6jXto48+MpLM5s2bHfXLUdhfffXVJjo62vz+++8lHtuzZ4+jJ/YHwv70sSXsDx48aDIyMhy3b1PYHz582BQWFhpjjLn22mvNZZdd5vW4r2HftWtXU7t2bZOVleWZNmPGDCPJLFq0yKe+3n777SY0NNT88ssvnmmLFy82kswLL7zgU5uTJ082kszq1as903744QcTGBhoxo4d61Obb7zxRol/4L/++quJjIw0N954o09trlq1ykgyjz32mGfa4cOHTWJiomnbtq1Pbe7YscMEBQWZ9PR0z7SioiJz+eWXm7p165qjR4/61O7WrVtNUVGRMcaYatWqnZawt3V9chr2+fn5pkaNGub+++931C9Hu/EzMjLUtGlTRUZGlngsJibG6/7MmTPVsWNHxcTEyO1268ILL9T06dNL1BXvDl62bJkuueQShYaGqnnz5lq2bJkkaf78+WrevLlCQkKUlJSkb7/91qu++Hjvzz//rNTUVFWrVk3x8fF68MEHZcoxoN/OnTt16623KjY2Vm63W02bNtXLL79c/oVynOJjm48//rimTZumRo0aqWrVqkpJSdH27dtljNFDDz2kunXrKjQ0VD169NBvv/3m1caCBQvUrVs3xcfHy+12KzExUQ899JAKCwtLPF/xc4SGhqp169b6/PPPSz1WnZeXp/Hjx6tx48Zyu92qV6+e7rnnHuXl5XnNt2/fPv344486dOiQT6//RJ9//rlnV17x844cOVKHDx8udf7yvIdFRUV66qmn1LRpU4WEhCg2NlZDhw7V77//7lMf9+7dq8aNG6tjx46aO3eujhw5ctKaWbNmqU+fPpKkK6+80nNsrXidlaSPP/5Yl19+uapVq6awsDB169ZN//vf/7zaKV53d+7cqZ49e6p69eqKjo7W6NGjS7zfr7/+upKSkhQWFqbw8HA1b95cTz/9tNc8P//8s/r06aOoqChVrVpVbdq00Ycffug1z7Jly+RyufT6669r3LhxqlOnjqpWrars7GwdOXJECxcuVOfOnZ0swlJlZ2dr8eLF6t+/v8LDwz3Tb7nlFlWvXl1vvvmmT+2+/fbbuuaaa1S/fn3PtM6dO+u8887zuc233npLrVq1UqtWrTzTmjRpok6dOp1Sm7Gxsbr++us906Kjo9W3b18tWLCgxGevvG0GBgZqyJAhnmkhISEaNGiQVqxYoe3btztuc8GCBSooKNAdd9zhmeZyuXT77bdrx44dWrFiheM2pWO7qF0ul0+1pTnX16fjBQUFqUOHDlqwYIGjOkdhn5CQoDVr1pTrxwHTp09XQkKC7rvvPv3rX/9SvXr1dMcdd2jatGkl5t28ebNuuukmde/eXZMmTdLvv/+u7t27a86cORo5cqT69++viRMnKiMjQ3379lVRUZFXfWFhobp06aLY2FhNmTJFSUlJGj9+vMaPH/+nfdyzZ4/atGmjTz75RMOGDdPTTz+txo0ba9CgQXrqqaecLBovc+bM0XPPPafhw4frH//4h5YvX66+fftq3LhxWrhwocaMGaMhQ4bo/fff1+jRo71qZ82aperVq2vUqFF6+umnlZSUpAceeED33nuv13zTp0/XsGHDVLduXU2ZMkWXX365evbsqR07dnjNV1RUpGuvvVaPP/64unfvrqlTp6pnz5568skndcMNN3jN++yzz+qCCy7Q6tWrfX7tx5s3b54OHTqk22+/XVOnTlVqaqqmTp2qW265pcS85X0Phw4dqrvvvluXXnqpnn76aQ0cOFBz5sxRamqqCgoKHPexdu3aevzxx7V3717dfPPNql27toYNG1biS+XxrrjiCt15552SpPvuu0+zZ8/W7NmzdcEFF0iSZs+erW7duql69eqaPHmy7r//fm3YsEGXXXZZiWP8hYWFSk1NVc2aNfX444+rffv2+te//qUXX3zRM8/ixYt14403qkaNGpo8ebIeffRRdejQQV9++aVnnj179qhdu3ZatGiR7rjjDj388MM6cuSIrr32Wr3zzjslXsNDDz2kDz/8UKNHj9Yjjzyi4OBgrVmzRvn5+frrX//qeDme6Pvvv9fRo0d1ySWXeE0PDg7WRRdd9KfLtyw7d+7Ur7/+WqJNSWrdurVPbRYVFWndunVltpmRkaGcnBzH7X777bf661//qoAA73+xrVu31qFDh/TTTz/51OZ5553nFXbFbUrHfvDlS5vVqlXzrLsntunLMj0TzvX16URJSUlav369srOzy1/kZDfA//3f/5nAwEATGBho2rZta+655x6zaNEik5+fX2LeQ4cOlZiWmppqGjVq5DUtISHBSDJfffWVZ9qiRYuMpBK7V1544YUSu07T0tKMJDN8+HDPtKKiItOtWzcTHBxs9u7d65muE3bjDxo0yNSuXbvEcZR+/fqZiIiIUl/DiX0/fld28e7O6Ohoc+DAAc/0sWPHGkmmZcuWpqCgwDP9xhtvNMHBwebIkSOeaaU959ChQ03VqlU98+Xl5ZmaNWuaVq1aebU3a9YsI8lr9/Xs2bNNQEBAiWN6zz//vJFkvvzyS8+08ePHl3vXdHl245f2WiZNmmRcLpfX+1re9/Dzzz83ksycOXO82ly4cGGJ6b4cs1+9erW57bbbTGRkpJFkLr74YjNt2rRSD1uVtRs/JyfHREZGmsGDB3tNz8zMNBEREV7Ti1/3gw8+6DXvxRdfbJKSkjz377rrLhMeHv6nu1RHjBhhJHm9zzk5OaZhw4amQYMGnt30xbsMGzVqVOL9eemll4wk8/3333tN92U3fvHy+eyzz0o81qdPHxMXF1fmaynLf//7XyPJvPrqqyUeu/vuu40kr89Seezdu7fU98AYY6ZNm2YkmR9//NFxX6tVq2ZuvfXWEtM//PBDI8ksXLjQcZtNmzY1HTt2LDH9f//7n5Fknn/+ecdtduvWrcT/ZGOMyc3NNZLMvffe67jNE52O3fg2r09Od+MbY8zcuXONJLNq1apy983Rlv1VV12lFStW6Nprr9V3332nKVOmKDU1VXXq1NF7773nNW9oaKjn76ysLO3bt0/t27fXzz//rKysLK95L7zwQrVt29ZzPzk5WZLUsWNHr90rxdNL+5XssGHDPH+7XC4NGzZM+fn5+uSTT0p9LcYYvf322+revbuMMdq3b5/nlpqaqqysLJ9/OdunTx9FRESU6Hf//v1VpUoVr+n5+fnauXOnZ9rxyy0nJ0f79u3T5ZdfrkOHDunHH3+UJH399dfav3+/Bg8e7NXezTffrBo1anj1Zd68ebrgggvUpEkTr9fYsWNHSdLSpUs9806YMEHGmNN2ytrxryU3N1f79u1Tu3btZIwp9Vvzyd7DefPmKSIiQldddZXXa0lKSlL16tW9XosvWrVqpenTp2v37t2aM2eOoqKiNGzYMNWuXVv9+/fXtm3bTtrG4sWLdeDAAd14441efQwMDFRycnKpfbztttu87l9++eVe63hkZKRyc3P/9FfXH330kVq3bq3LLrvMM6169eoaMmSItm7dqg0bNnjNn5aW5vX+SNL+/fslqcQ65IviQzVut7vEYyEhIWUeyjmVNo+fx59tFtecq22eCef6+nSi4s/ovn37yl1T5eSzeGvVqpXmz5+v/Px8fffdd3rnnXf05JNPqnfv3lq7dq0uvPBCSdKXX36p8ePHa8WKFSWOAWdlZXmF4fGBLsnzWL169UqdfuLx2YCAADVq1Mhr2nnnnSdJZZ4atXfvXh04cEAvvvii1y7T4/3666+lTj+ZU3k9//vf/zRu3Dh9+umnJXbRFH9J+uWXXyRJjRs39nq8SpUqatCggde0TZs26YcfflB0dHSpffX1NZbHtm3b9MADD+i9994r8Z6d+IWvPO/hpk2blJWVVeL3IcX+7LVkZmZ63Y+IiCgRdsVCQkJ00003qW/fvpo+fbpGjx6tOXPmqHfv3iXe2xNt2rRJkjxfpk504i7YkJCQEu9NjRo1vJbXHXfcoTfffFNdu3ZVnTp1lJKSor59+6pLly6eeX755RfPl8rjFe+e/eWXX7zO123YsGGZr8GU47cuJ1O8bEs7Nn3kyJEyl/2ptHn8PP5ss7jmXG3zTDjX16cTFX9GnfwuwnHYFwsODvb8COG8887TwIEDNW/ePI0fP14ZGRnq1KmTmjRpoieeeEL16tVTcHCwPvroIz355JMljrkHBgaW+hxlTT8d/4yK+9C/f3+lpaWVOk+LFi18atvX13PgwAG1b99e4eHhevDBB5WYmKiQkBB98803GjNmTInlVh5FRUVq3ry5nnjiiVIfP/ELyOlSWFioq666Sr/99pvGjBmjJk2aqFq1atq5c6cGDBjg82uJiYnRnDlzSn28rC800rFj88ebOXNmmef+/vDDD5o5c6Zmz56tzMxMNW3aVIMGDdKVV15Zrj5Kx47bx8XFlXj8+D0xUtnrxPFiYmK0du1aLVq0SB9//LE+/vhjzZw5U7fccoteeeWVk9aXprR/ODVr1pR07Mtn3bp1fWq3WPHy3r17d4nHdu/erfj4+NPeZlRUVKlbVH+muKasNiX53Ncz0ebxewFPV5tLly6VMcYrOE6lzTPhXF+fTlS8MVCrVq1y1/gc9scr/jFC8Yt5//33lZeXp/fee89rS+hUd7OWpaioSD///LNnS1CS5wcwJ27pFouOjlZYWJgKCwtPy6+PT4dly5Zp//79mj9/vq644grP9C1btnjNl5CQIOnYDxuPD6CjR49q69atXl9SEhMT9d1336lTp06n9dexJ/P999/rp59+0iuvvOL1g7yydkWX5z1MTEzUJ598oksvvdTxt+MTn7dp06Ze97OysvTGG2/o5Zdf1qpVq1S9enXdcMMN+vvf/642bdqUaK+sZZmYmCjpWECfzvUqODhY3bt3V/fu3VVUVKQ77rhDL7zwgu6//341btxYCQkJ2rhxY4m64kM/xevMnym+gMeWLVvUvHnzU+pvs2bNVKVKFX399dfq27evZ3p+fr7Wrl3rNa286tSpo+joaH399dclHlu9erUuuugix20GBASoefPmpba5atUqNWrUSGFhYY7bveiii/T555+rqKjI60d6q1atUtWqVb3WcydtLl26VNnZ2V57iFatWuV53Jc2X3rpJf3www+evbKn2uaZcK6vTyfasmWLAgICHK1Hjo7ZF38DPNFHH30kSTr//PMl/bG1cvy8WVlZmjlzppOnc+TZZ5/1/G2M0bPPPqugoCB16tSp1PkDAwPVq1cvvf3226WeXXCqV1DyRWnLLT8/X88995zXfJdccolq1qypGTNm6OjRo57pc+bMKbG7vG/fvtq5c6dmzJhR4vkOHz6s3Nxcz/3Teepdaa/FGFPidLHjnew97Nu3rwoLC/XQQw+VqD169KgOHDhQZtudO3f2uhV/q8/JyVH//v1Vu3ZtDR06VC6XSy+99JJ2796tl156qdSgl6Rq1apJUonnTE1NVXh4uB555JFSzw7wZb0qPpZeLCAgwPOFrnh34dVXX63Vq1d7nSqVm5urF198UQ0aNPD6R16WpKQkBQcHl/qPyqmIiAh17txZ//nPf7x+fTx79mwdPHjQc+qiU7169dIHH3zgdZrZkiVL9NNPP/ncZu/evfXf//7X63Vv3LhRn3766Sm1uWfPHs2fP98zbd++fZo3b566d+/ueIuxuM3CwkKvw455eXmaOXOmkpOTfdpL16NHDwUFBXn9jzHG6Pnnn1edOnXUrl07x22eCef6+nSiNWvWqGnTpl6Hw0/G0Zb98OHDdejQIV133XVq0qSJ8vPz9dVXX+mNN95QgwYNNHDgQElSSkqKZ0tk6NChOnjwoGbMmKGYmJhSd2+cqpCQEC1cuFBpaWlKTk7Wxx9/rA8//FD33Xffn+7affTRR7V06VIlJydr8ODBuvDCC/Xbb7/pm2++0SeffFLiHPgzrV27dqpRo4bS0tJ05513yuVyafbs2SW+YAUHB2vChAkaPny4OnbsqL59+2rr1q2aNWuWEhMTvbY6//a3v+nNN9/UbbfdpqVLl+rSSy9VYWGhfvzxR7355ptatGiRZ8/Ms88+q4kTJ2rp0qXl+pHe5s2b9c9//rPE9IsvvlgpKSlKTEzU6NGjtXPnToWHh+vtt98u83z48ryH7du319ChQzVp0iStXbtWKSkpCgoK0qZNmzRv3jw9/fTT6t27d3kXt6RjQbpo0SLddtttGjRoUIkt/rJcdNFFCgwM1OTJk5WVlSW32+25rsT06dP1t7/9TX/961/Vr18/RUdHa9u2bfrwww916aWXen2pKY+///3v+u2339SxY0fVrVtXv/zyi6ZOnaqLLrrIc0z+3nvv1WuvvaauXbvqzjvvVFRUlF555RVt2bJFb7/9dolTwEoTEhKilJQUffLJJ3rwwQcd9bE0Dz/8sNq1a6f27dtryJAh2rFjh/71r38pJSXF6/cG0rE9Je3bt/e6VkFp7rvvPs2bN09XXnml7rrrLh08eFCPPfaYmjdv7vn/U6x4j9DJLml8xx13aMaMGerWrZtGjx6toKAgPfHEE4qNjdU//vEPr3k7dOig5cuXn/RQYu/evdWmTRsNHDhQGzZsUK1atfTcc8+psLBQEydO9Jp3wIABnveqrD2R0rEf9Pbp00djx47Vr7/+qsaNG+uVV17R1q1b9e9//9tr3gkTJpTrs1y3bl2NGDFCjz32mAoKCtSqVSu9++67+vzzzzVnzhyvw0yzZs3SwIED//QQWLH3339f3333nSSpoKBA69at8/yvuPbaaz1fVrdu3aqGDRsqLS1Ns2bN+tM2bVyffFFQUKDly5d7XRuhXMr9u31jzMcff2xuvfVW06RJE1O9enXPpXOHDx9e4gp67733nmnRooUJCQkxDRo0MJMnTzYvv/yykWS2bNnima+sU7gkeV3VyZg/TgE6/gpSxVdfy8jIMCkpKaZq1aomNjbWjB8/3nO60fFtnngFvT179pj09HRTr149ExQUZOLi4kynTp3Miy++eNLlUdapd8f3z5iyT62YOXOmkWT++9//eqZ9+eWXpk2bNiY0NNTEx8d7Tm9UKad5PfPMMyYhIcG43W7TunVr8+WXX5qkpKQSl4/Mz883kydPNk2bNjVut9vUqFHDJCUlmYkTJ3pdjcrpqXeSSr0NGjTIGGPMhg0bTOfOnU316tVNrVq1zODBg813331X4jQuJ++hMca8+OKLJikpyYSGhpqwsDDTvHlzc88995hdu3Z55invqXf5+fk+XxZzxowZplGjRiYwMLDEclu6dKlJTU01ERERJiQkxCQmJpoBAwaYr7/+usTrPlHx+1DsrbfeMikpKSYmJsYEBweb+vXrm6FDh5rdu3d71WVkZJjevXubyMhIExISYlq3bm0++OADr3n+7DQfY4yZP3++cblcZtu2bZ5pvl5Bz5hjp0u2a9fOhISEmOjoaJOenm6ys7O95snJyTGSTL9+/cps53jr16/3rCeRkZHm5ptvNpmZmSXmq1WrlmnTpk252ty+fbvp3bu3CQ8PN9WrVzfXXHON2bRpU4n5kpKSyn2a12+//WYGDRpkatasaapWrWrat2/v9Vkv1qtXLxMaGlrqKZ4nOnz4sBk9erSJi4szbrfbtGrVqtTT+P7xj38Yl8tlfvjhh5O2WVhYaB555BGTkJBggoODTdOmTc1//vOfEvNNnTq13KcNFp9WWtrt+PXo+++/d3SKn23rkzHOT737+OOPjaQy2yuLo7A/G52JS61WVoWFhSYqKsr8/e9/93dXUEkdPXrUnHfeeWbcuHGeacVhP3XqVLN3716vL0dpaWmmXr16Zu/eveUKq9J8+OGHxuVymXXr1p1q9z2Kzz0/8cvOqcjOzjZVqlQxzz777Glr0xhjYmJizOjRo09rm61atTK9e/c+rW326dPHtGrV6rS2OW3aNFOtWrVSw9VXlWV9Onr0qNm7d6959913S4R9dna22bt3r2nXrl2JsO/Ro4fp2bOn4+cj7Cupw4cPe649Xax4T0Fp38qB8nr99ddNjRo1TE5OjjHmj7Avvh3/T+n4LbjSLv5RHqNHj/b5evFlefbZZ32+XnxZPvjgA5OQkODznqDSrF+/3oSFhXld/OtUZWVlmeDgYLNhw4bT1mZRUZGJjo72+Rr0Zendu7fP4w+UpbKsT99++22Zn6sePXqU+rnasGGDCQwMLHHhq/JwGXMazmPzowEDBuitt9465WEOK5tly5Zp5MiR6tOnj2rWrKlvvvlG//73v3XBBRdozZo1Cg4O9ncXYYkjR47oiy++8Nxv0aKF51oHGzZs0K5duyQdu4hPWT9oBODt4MGDWrlypef+8Z+rdevWea4bcro+V4R9JbV161bdeeedWr16tX777TdFRUXp6quv1qOPPlrmRWcAAOemSh/2AADgzzk6zx4AAFQ+hD0AAJY7LZfLtUlRUZF27dqlsLCwCr28LADg9DDGKCcnR/Hx8eW6oNS5gLA/wa5du87Y4DAAgIqzffv2Ux7UyRaE/QlOxyAFqBjlGTHuRIWFhWegJyUFBQX5VHf8WAfl5ctvbH1Zdr6oqOUt/TFegRPHjw1xJvm6l9CX9zYyMtJxzZ+NK1GZ8f/8D1bu35g2bZoaNGigkJAQJScna/Xq1eWuZdd95eFyuRzfzua++Xo7m/t3ti+/inK2v7e2svm1OWVd2L/xxhsaNWqUxo8fr2+++UYtW7ZUamqq5wIFAACca6wL+yeeeEKDBw/WwIEDdeGFF+r5559X1apV9fLLL/u7awAA+IVVYZ+fn681a9aoc+fOnmkBAQHq3Lmz1zjfx8vLy1N2drbXDQAAm1gV9vv27VNhYaFiY2O9psfGxiozM7PUmkmTJikiIsJz45f4AADbWBX2vhg7dqyysrI8t+3bt/u7SwAAnFZWnXpXq1YtBQYGas+ePV7T9+zZo7i4uFJr3G633G53RXQPAAC/sGrLPjg4WElJSVqyZIlnWlFRkZYsWaK2bdv6sWcAAPiPVVv2kjRq1CilpaXpkksuUevWrfXUU08pNzdXAwcO9HfXAADwC+vC/oYbbtDevXv1wAMPKDMzUxdddJEWLlxY4kd7AACcKxjP/gTZ2dmKiIjwdzdwhgQHB1dIzcGDBx3X+MqXS/P6MjhIXl6e4xpf1alTx3HNzp07HddUqeJ8e8eXSxr7etlWX9a9/fv3+/RcNsrKylJ4eLi/u3FWsOqYPQAAKImwBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwnHWj3gF/pqCgwHFNYWGh4xpfBpqRpKKiIsc1vrymiuJyuXyq82VQG1+43W7HNb4MhJOTk+O4xle+DPySnZ19BnqCswlb9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlGPUOlZYvI5bl5eU5rvFl1LsqVXz7aIWEhDiuOXTokE/P5VRoaKjjmsDAQJ+eKz8/33HNXXfd5bhm+PDhjmvq1avnuMYY47hG8m3UwMjISJ+eC3Zjyx4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAlmMgHFRavgxq48tgLkeOHHFcc/ToUcc1klRUVOS4JigoyHFNQUGB45rDhw87rvHVyJEjHdcMGzbMcU3NmjUd16xevdpxzbx58xzXSNL555/vuCYlJcVxja/9Q+XBlj0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALMdAOKi0XC6X4xpfBoAxxjiu8VVAgPPv3768Jl8kJiY6rrnrrrt8eq7+/fs7rvFlkKNRo0Y5rpk+fbrjGl/6Jvm2PuTm5vr0XLAbW/YAAFiOsAcAwHLWhf2ECRPkcrm8bk2aNPF3twAA8Bsrj9k3bdpUn3zyied+lSpWvkwAAMrFyhSsUqWK4uLi/N0NAADOCtbtxpekTZs2KT4+Xo0aNdLNN9+sbdu2lTlvXl6esrOzvW4AANjEurBPTk7WrFmztHDhQk2fPl1btmzR5ZdfrpycnFLnnzRpkiIiIjy3evXqVXCPAQA4s6wL+65du6pPnz5q0aKFUlNT9dFHH+nAgQN68803S51/7NixysrK8ty2b99ewT0GAODMsvKY/fEiIyN13nnnafPmzaU+7na75Xa7K7hXAABUHOu27E908OBBZWRkqHbt2v7uCgAAfmFd2I8ePVrLly/X1q1b9dVXX+m6665TYGCgbrzxRn93DQAAv7BuN/6OHTt04403av/+/YqOjtZll12mlStXKjo62t9dAwDAL1ymIkf5qASys7MVERHh726gHIKCghzXVNSgMb4OfFJUVOS4Ji8vz3FN06ZNHdfcc889jmtuvvlmxzWS9N133zmuueaaaxzX7N6923FNYGBghdRIUn5+vk91OCYrK0vh4eH+7sZZwbrd+AAAwBthDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOetGvcO5w5dBbXwZoKawsNBxzeHDhx3X+MqXgT6uu+46xzW33HKL45rPPvvMcY0k3XrrrY5rfBnUxpfBlHwZrKgiB7TxZR2vyPUV/sGWPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5Rr3DOcWX0b0CAwMd17jdbsc1km+jo40dO9Zxzb333uu4Zvny5Y5r+vXr57hGknJychzXBAQ433bxZURDX57H5XI5rpEkY4zjGkawQ2nYsgcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5RgIB5VWSEiI4xpfBiTxZWARXwZYkXwb1GbMmDGOa7Kzsx3XpKenO67JzMx0XHO283VQG8Cf2LIHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUYCAeV1pEjRyrkeQIDAx3XREZG+vRcHTp0cFyTm5vruGbEiBGOa3bt2uW4Jjg42HGNJBUVFflU51RAgPPtHWOM4xpfB0YCThe27AEAsBxhDwCA5SpV2H/22Wfq3r274uPj5XK59O6773o9bozRAw88oNq1ays0NFSdO3fWpk2b/NNZAADOEpUq7HNzc9WyZUtNmzat1MenTJmiZ555Rs8//7xWrVqlatWqKTU1tcKO7QIAcDaqVD/Q69q1q7p27VrqY8YYPfXUUxo3bpx69OghSXr11VcVGxurd999V/369avIrgIAcNaoVFv2f2bLli3KzMxU586dPdMiIiKUnJysFStWlFmXl5en7OxsrxsAADaxJuwzMzMlSbGxsV7TY2NjPY+VZtKkSYqIiPDc6tWrd0b7CQBARbMm7H01duxYZWVleW7bt2/3d5cAADitrAn7uLg4SdKePXu8pu/Zs8fzWGncbrfCw8O9bgAA2MSasG/YsKHi4uK0ZMkSz7Ts7GytWrVKbdu29WPPAADwr0r1a/yDBw9q8+bNnvtbtmzR2rVrFRUVpfr162vEiBH65z//qb/85S9q2LCh7r//fsXHx6tnz57+6zQAAH5WqcL+66+/1pVXXum5P2rUKElSWlqaZs2apXvuuUe5ubkaMmSIDhw4oMsuu0wLFy5USEiIv7oMAIDfuYwvozpYLDs7WxEREf7uBsrB7XY7rvFlYJacnBzHNU8//bTjGkm68847HdfMnz/fcU2vXr0c16DiuVwuxzVVqjjfhisoKHBcUxlkZWXxO6z/nzXH7AEAQOkIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjlHvTsCod3aLjo52XOPL6Ho///yz4xpJ2rRpk+OalJQUxzU7d+50XHO282WEOBv//bEc/sCod39gyx4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAlqvi7w4AvgoLC3Ncs3fvXsc1Y8eOdVwTFBTkuEaSFixY4LjGl0FtfFl2OTk5jmt8FRDgfDvElwFgioqKHNf4wpe+Sb71z9ZBbXBq2LIHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUYCAeVli8DsyQnJzuuufvuux3XFBYWOq6RpOzsbJ/qnDpy5IjjGrfb7bgmPz/fcY1UcQPUVBRfBvaRpODgYMc1viy7o0ePOq5B5cKWPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAs5zLGGH934mySnZ2tiIgIf3cD5dCmTRvHNYsWLXJcExQU5LjG149VaGio45off/zRcc3evXsd17z77ruOa7Zs2eK4RpIOHDjguGb79u2Oa3bs2OG4Ji8vz3EN/CMrK0vh4eH+7sZZgS17AAAsR9gDAGC5ShX2n332mbp37674+Hi5XK4SuxUHDBggl8vldevSpYt/OgsAwFmiUoV9bm6uWrZsqWnTppU5T5cuXbR7927P7bXXXqvAHgIAcPap4u8OONG1a1d17dr1T+dxu92Ki4uroB4BAHD2q1Rb9uWxbNkyxcTE6Pzzz9ftt9+u/fv3/+n8eXl5ys7O9roBAGATq8K+S5cuevXVV7VkyRJNnjxZy5cvV9euXVVYWFhmzaRJkxQREeG51atXrwJ7DADAmVepduOfTL9+/Tx/N2/eXC1atFBiYqKWLVumTp06lVozduxYjRo1ynM/OzubwAcAWMWqLfsTNWrUSLVq1dLmzZvLnMftdis8PNzrBgCATawO+x07dmj//v2qXbu2v7sCAIDfVKrd+AcPHvTaSt+yZYvWrl2rqKgoRUVFaeLEierVq5fi4uKUkZGhe+65R40bN1Zqaqofew0AgH9VqrD/+uuvdeWVV3ruFx9rT0tL0/Tp07Vu3Tq98sorOnDggOLj45WSkqKHHnpIbrfbX10GAMDvGAjnBAyEU3mMGzfOcc1DDz10BnpS0q5du3yq+7MzR8oSExPjuMaXL8AFBQWOawICfDtSePToUcc1W7dudVwzcuRIxzW+DKZUVFTkuEby7X1ioJ4/MBDOH6w+Zg8AAAh7AACsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiuUg1xCxxv7969jmtWrlzpuCY5OdlxzRdffOG4RpKGDBniuKZOnTqOa+Lj4x3XDBs2zHFNjx49HNdIvo2Wd/755zuuefjhhx3XLF++3HHN4cOHHddIjGCH04ctewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjoFwUGm9+uqrjmuaNGniuKZ169aOa3wZPEeSoqOjHdds2LDBcc22bdsc13z11VeOawoKChzXSFKXLl0c18yaNctxTcOGDR3XNGjQwHGNL++RJIWHhzuuyc7O9um5YDe27AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOZcxxvi7E2eT7OxsRURE+LsbOEOaNWvmuOabb75xXBMUFOS4RpIeeeQRxzUTJkxwXOPrADVO1alTx6e6nTt3Oq4ZPHiw45oXXnjBcY0vgyn99NNPjmt8FRgY6LimsLDwDPTE/7KysnwaTMhGbNkDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHJV/N0BwFeRkZGOa9avX++45sEHH3RcM2bMGMc1kjRgwADHNW+88YbjmnXr1jmuqVq1quMaXwa08VV8fLzjGpfL5bgmPz/fcY2v3G6345q8vLwz0BNUdmzZAwBgOcIeAADLVaqwnzRpklq1aqWwsDDFxMSoZ8+e2rhxo9c8R44cUXp6umrWrKnq1aurV69e2rNnj596DACA/1WqsF++fLnS09O1cuVKLV68WAUFBUpJSVFubq5nnpEjR+r999/XvHnztHz5cu3atUvXX3+9H3sNAIB/Vaof6C1cuNDr/qxZsxQTE6M1a9boiiuuUFZWlv79739r7ty56tixoyRp5syZuuCCC7Ry5Uq1adPGH90GAMCvKtWW/YmysrIkSVFRUZKkNWvWqKCgQJ07d/bM06RJE9WvX18rVqwotY28vDxlZ2d73QAAsEmlDfuioiKNGDFCl156qZo1ayZJyszMVHBwcIlTsmJjY5WZmVlqO5MmTVJERITnVq9evTPddQAAKlSlDfv09HStX79er7/++im1M3bsWGVlZXlu27dvP009BADg7FCpjtkXGzZsmD744AN99tlnqlu3rmd6XFyc8vPzdeDAAa+t+z179iguLq7Uttxut08XrgAAoLKoVFv2xhgNGzZM77zzjj799FM1bNjQ6/GkpCQFBQVpyZIlnmkbN27Utm3b1LZt24ruLgAAZ4VKtWWfnp6uuXPnasGCBQoLC/Mch4+IiFBoaKgiIiI0aNAgjRo1SlFRUQoPD9fw4cPVtm1bfokPADhnVaqwnz59uiSpQ4cOXtNnzpzpuab4k08+qYCAAPXq1Ut5eXlKTU3Vc889V8E9BQDg7OEyxhh/d+Jskp2drYiICH93A+UQEOD8KFRQUJDjGl8GFnn++ecd10jS0KFDHdf8/vvvjmu2bdvmuGbOnDmOay644ALHNZLUp08fxzW+DNTz6quvOq4ZNWqU4xpf3iPJt4FwCgoKHNcUFRU5rqkMsrKyFB4e7u9unBUq1TF7AADgHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsx6t0JGPXObi6Xy3GNLyPl+cqXUe+eeeaZM9CTknbs2OG4ZsaMGT49V1RUlOOacePGOa45ePCg45rAwEDHNXXq1HFcI/k2OmGVKs5HLj969KjjmsqAUe/+wJY9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACzHQDgnYCCcyiMgwPl31aKiIsc1oaGhjmsOHz7suEaS3G6345q8vDyfnsspXwYRqsh/L2fzIEf5+fk+1fkyiEt2drZPz2UjBsL5A1v2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALBcFX93APCVL4Pa+MLXQW18UVGD2vjibB8zy5f++TpATUVhUBucLmzZAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5SpV2E+aNEmtWrVSWFiYYmJi1LNnT23cuNFrng4dOsjlcnndbrvtNj/1GAAA/6tUYb98+XKlp6dr5cqVWrx4sQoKCpSSkqLc3Fyv+QYPHqzdu3d7blOmTPFTjwEA8L8q/u6AEwsXLvS6P2vWLMXExGjNmjW64oorPNOrVq2quLi4iu4eAABnpUq1ZX+irKwsSVJUVJTX9Dlz5qhWrVpq1qyZxo4dq0OHDpXZRl5enrKzs71uAADYpFJt2R+vqKhII0aM0KWXXqpmzZp5pt90001KSEhQfHy81q1bpzFjxmjjxo2aP39+qe1MmjRJEydOrKhuAwBQ4VzGGOPvTvji9ttv18cff6wvvvhCdevWLXO+Tz/9VJ06ddLmzZuVmJhY4vG8vDzl5eV57mdnZ6tevXpnpM8AgIqTlZWl8PBwf3fjrFApt+yHDRumDz74QJ999tmfBr0kJScnS1KZYe92u+V2u89IPwEAOBtUqrA3xmj48OF65513tGzZMjVs2PCkNWvXrpUk1a5d+wz3DgCAs1OlCvv09HTNnTtXCxYsUFhYmDIzMyVJERERCg0NVUZGhubOnaurr75aNWvW1Lp16zRy5EhdccUVatGihZ97DwCAf1SqY/Yul6vU6TNnztSAAQO0fft29e/fX+vXr1dubq7q1aun6667TuPGjSv3cZvs7GxFRESczm4DAPyAY/Z/qFRhXxEIewCwA2H/h0p9nj0AADg5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALEfYAwBgOcIeAADLEfYAAFiOsAcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2AMAYDnCHgAAyxH2AABYjrAHAMByhD0AAJYj7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wv4Exhh/dwEAcBrw//wPhP0JcnJy/N0FAMBpwP/zP7gMX328FBUVadeuXQoLC5PL5fJ6LDs7W/Xq1dP27dsVHh7upx76H8vhGJbDMSyHY1gOx5wNy8EYo5ycHMXHxysggG1aSari7w6cbQICAlS3bt0/nSc8PPyc/jAXYzkcw3I4huVwDMvhGH8vh4iICL8999mIrzwAAFiOsAcAwHKEvQNut1vjx4+X2+32d1f8iuVwDMvhGJbDMSyHY1gOZyd+oAcAgOXYsgcAwHKEPQAAliPsAQCwHGEPAIDlCHsAACxH2JfTtGnT1KBBA4WEhCg5OVmrV6/2d5cq1IQJE+RyubxuTZo08Xe3zrjPPvtM3bt3V3x8vFwul959912vx40xeuCBB1S7dm2Fhoaqc+fO2rRpk386ewadbDkMGDCgxPrRpUsX/3T2DJo0aZJatWqlsLAwxcTEqGfPntq4caPXPEeOHFF6erpq1qyp6tWrq1evXtqzZ4+fenxmlGc5dOjQocQ6cdttt/mpxyDsy+GNN97QqFGjNH78eH3zzTdq2bKlUlNT9euvv/q7axWqadOm2r17t+f2xRdf+LtLZ1xubq5atmypadOmlfr4lClT9Mwzz+j555/XqlWrVK1aNaWmpurIkSMV3NMz62TLQZK6dOnitX689tprFdjDirF8+XKlp6dr5cqVWrx4sQoKCpSSkqLc3FzPPCNHjtT777+vefPmafny5dq1a5euv/56P/b69CvPcpCkwYMHe60TU6ZM8VOPIYOTat26tUlPT/fcLywsNPHx8WbSpEl+7FXFGj9+vGnZsqW/u+FXksw777zjuV9UVGTi4uLMY4895pl24MAB43a7zWuvveaHHlaME5eDMcakpaWZHj16+KU//vTrr78aSWb58uXGmGPvf1BQkJk3b55nnh9++MFIMitWrPBXN8+4E5eDMca0b9/e3HXXXf7rFLywZX8S+fn5WrNmjTp37uyZFhAQoM6dO2vFihV+7FnF27Rpk+Lj49WoUSPdfPPN2rZtm7+75FdbtmxRZmam17oRERGh5OTkc27dkKRly5YpJiZG559/vm6//Xbt37/f310647KysiRJUVFRkqQ1a9aooKDAa51o0qSJ6tevb/U6ceJyKDZnzhzVqlVLzZo109ixY3Xo0CF/dA9i1LuT2rdvnwoLCxUbG+s1PTY2Vj/++KOfelXxkpOTNWvWLJ1//vnavXu3Jk6cqMsvv1zr169XWFiYv7vnF5mZmZJU6rpR/Ni5okuXLrr++uvVsGFDZWRk6L777lPXrl21YsUKBQYG+rt7Z0RRUZFGjBihSy+9VM2aNZN0bJ0IDg5WZGSk17w2rxOlLQdJuummm5SQkKD4+HitW7dOY8aM0caNGzV//nw/9vbcRdijXLp27er5u0WLFkpOTlZCQoLefPNNDRo0yI89w9mgX79+nr+bN2+uFi1aKDExUcuWLVOnTp382LMzJz09XevXrz8nfrvyZ8paDkOGDPH83bx5c9WuXVudOnVSRkaGEhMTK7qb5zx2459ErVq1FBgYWOLXtHv27FFcXJyfeuV/kZGROu+887R582Z/d8Vvit9/1o2SGjVqpFq1alm7fgwbNkwffPCBli5dqrp163qmx8XFKT8/XwcOHPCa39Z1oqzlUJrk5GRJsnadONsR9icRHByspKQkLVmyxDOtqKhIS5YsUdu2bf3YM/86ePCgMjIyVLt2bX93xW8aNmyouLg4r3UjOztbq1atOqfXDUnasWOH9u/fb936YYzRsGHD9M477+jTTz9Vw4YNvR5PSkpSUFCQ1zqxceNGbdu2zap14mTLoTRr166VJOvWicqC3fjlMGrUKKWlpemSSy5R69at9dRTTyk3N1cDBw70d9cqzOjRo9W9e3clJCRo165dGj9+vAIDA3XjjTf6u2tn1MGDB722RLZs2aK1a9cqKipK9evX14gRI/TPf/5Tf/nLX9SwYUPdf//9io+PV8+ePf3X6TPgz5ZDVFSUJk6cqF69eikuLk4ZGRm655571LhxY6Wmpvqx16dfenq65s6dqwULFigsLMxzHD4iIkKhoaGKiIjQoEGDNGrUKEVFRSk8PFzDhw9X27Zt1aZNGz/3/vQ52XLIyMjQ3LlzdfXVV6tmzZpat26dRo4cqSuuuEItWrTwc+/PUf4+HaCymDp1qqlfv74JDg42rVu3NitXrvR3lyrUDTfcYGrXrm2Cg4NNnTp1zA033GA2b97s726dcUuXLjWSStzS0tKMMcdOv7v//vtNbGyscbvdplOnTmbjxo3+7fQZ8GfL4dChQyYlJcVER0eboKAgk5CQYAYPHmwyMzP93e3TrrRlIMnMnDnTM8/hw4fNHXfcYWrUqGGqVq1qrrvuOrN7927/dfoMONly2LZtm7niiitMVFSUcbvdpnHjxubuu+82WVlZ/u34OYzx7AEAsBzH7AEAsBxhDwCA5Qh7AAAsR9gDAGA5wh4AAMsR9gAAWI6wBwDAcoQ9AACWI+wBALAcYQ8AgOUIewAALPf/AeGArbugNgpTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_features, train_labels = next(iter(train_dataloader))\n",
    "img = train_features[0].squeeze()\n",
    "label = train_labels[0]\n",
    "plt.title(f\"Sample Image: Label->{label}\")\n",
    "plt.imshow(img.reshape(28,28), cmap=\"gray\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ad678500",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature batch shape: torch.Size([64, 784])\n",
      "Labels batch shape: torch.Size([64, 1, 10])\n"
     ]
    }
   ],
   "source": [
    "print(f\"Feature batch shape: {train_features.size()}\")\n",
    "print(f\"Labels batch shape: {train_labels.size()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fed589d",
   "metadata": {},
   "source": [
    "#### Simple Neural Network for Predicting Class of HandWritten Digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8361b5dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, h1, h2):\n",
    "        super(NeuralNet, self).__init__() #Initializing superclass\n",
    "        self.layer1 = nn.Linear(784, h1) #784 is the number of input features\n",
    "        self.layer2 = nn.Linear(h1, h2)\n",
    "        self.layer3 = nn.Linear(h2, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        out = F.log_softmax(x, dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14225387",
   "metadata": {},
   "source": [
    "### Training Loop in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f5e6cdc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: 1.8479397296905518\n",
      "1: 0.9750990271568298\n",
      "2: 0.6475524306297302\n",
      "3: 0.5225962996482849\n",
      "4: 0.45880556106567383\n",
      "5: 0.4211426377296448\n",
      "6: 0.3965868055820465\n",
      "7: 0.37958985567092896\n",
      "8: 0.3659931421279907\n",
      "9: 0.3560301959514618\n"
     ]
    }
   ],
   "source": [
    "# Constants\n",
    "lr = 0.005\n",
    "epochs = 10\n",
    "\n",
    "# Loading Dataset\n",
    "train_dataset = CustomMNISTDataset('./output/training/')\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# Initialize Model\n",
    "model = NeuralNet(512, 128)\n",
    "# Initialize Optimizer\n",
    "optimizer = optim.Adadelta(model.parameters(), lr=0.005)\n",
    "# Loss function\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "# set model to train mode\n",
    "losses = []\n",
    "model.train()\n",
    "for epoch in range(epochs):\n",
    "\tbatch_losses = []    \n",
    "\tfor batch_idx, (data, targets) in enumerate(train_dataloader):\n",
    "\t\toptimizer.zero_grad()\n",
    "\t\toutputs = model(data)\n",
    "\t\tloss = loss_fn(outputs, targets.squeeze(1).float())\n",
    "\t\tloss.backward() # Backpropagation of loss\n",
    "\t\toptimizer.step() # Updating Weights based on Gradients\n",
    "\t\tbatch_losses.append(loss.item())\n",
    "\tfinal_epoch_loss = torch.mean(torch.Tensor(batch_losses))\n",
    "\tlosses.append(final_epoch_loss)\n",
    "\tprint(f\"{epoch}: {final_epoch_loss}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c4ac119",
   "metadata": {},
   "source": [
    "#### Plotting Training Loss and Epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "152696c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRbElEQVR4nO3deVxU5eIG8OfMwMywDptsioKguCO5IFpZaSoqZVmZWqK23EozRetqpaal/LJSb2naYsu9bS6VVm6ZpqZprqi5oQKCKCAiDOsAM+f3BzA6gcjoDGeW5/v5zEfmzDnDg6g8vu97zhFEURRBREREZCdkUgcgIiIiMieWGyIiIrIrLDdERERkV1huiIiIyK6w3BAREZFdYbkhIiIiu8JyQ0RERHaF5YaIiIjsCssNERER2RWWGyI7MHbsWISGht7SsW+88QYEQTBvILIroaGhGDp0qNQxiBqN5YbIggRBaNRj+/btUkeVxNixY+Hu7i51DMmFhobe8M/GoEGDpI5HZHOcpA5AZM/+97//GT3/73//iy1bttTZ3r59+9v6PJ988gn0ev0tHfv6669j+vTpt/X56fZ17doVU6dOrbM9ODhYgjREto3lhsiCnnjiCaPne/fuxZYtW+ps/6fS0lK4uro2+vM4OzvfUj4AcHJygpMT/ymQWvPmzW/654KIGofTUkQSu+eee9CpUyccPHgQd999N1xdXfHqq68CANatW4chQ4YgODgYSqUS4eHhePPNN6HT6Yze459rbtLT0yEIAt599118/PHHCA8Ph1KpRI8ePbB//36jY+tbcyMIAiZOnIi1a9eiU6dOUCqV6NixIzZt2lQn//bt29G9e3eoVCqEh4fjo48+Mvs6ntWrV6Nbt25wcXGBn58fnnjiCWRlZRntk52djXHjxqFFixZQKpUICgrCgw8+iPT0dMM+Bw4cwMCBA+Hn5wcXFxeEhYVh/PjxDX7uoUOHonXr1vW+Fhsbi+7duxueb9myBXfeeSe8vLzg7u6OyMhIw/fSHGqn8VJTUzFw4EC4ubkhODgYc+fOhSiKRvuWlJRg6tSpCAkJgVKpRGRkJN599906+wHAV199hZ49e8LV1RXe3t64++678euvv9bZb9euXejZsydUKhVat26N//73v2b72ojMif9dI7ICV65cQVxcHB5//HE88cQTCAgIAAB88cUXcHd3R2JiItzd3bFt2zbMmjULGo0G77zzzk3f95tvvkFRURH+9a9/QRAELFiwAA8//DBSU1NvOtqza9cu/PDDD3jhhRfg4eGB999/H8OHD0dGRgZ8fX0BAIcPH8agQYMQFBSEOXPmQKfTYe7cuWjWrNnt/6bU+OKLLzBu3Dj06NEDSUlJyMnJwX/+8x/s3r0bhw8fhpeXFwBg+PDhOH78OF588UWEhoYiNzcXW7ZsQUZGhuH5gAED0KxZM0yfPh1eXl5IT0/HDz/80ODnHzFiBMaMGYP9+/ejR48ehu3nz5/H3r17Dd+H48ePY+jQoejSpQvmzp0LpVKJs2fPYvfu3Y36OisrK5GXl1dnu5ubG1xcXAzPdTodBg0ahF69emHBggXYtGkTZs+ejaqqKsydOxcAIIoiHnjgAfz+++946qmn0LVrV2zevBkvv/wysrKysGjRIsP7zZkzB2+88QZ69+6NuXPnQqFQ4K+//sK2bdswYMAAw35nz57FI488gqeeegoJCQn47LPPMHbsWHTr1g0dO3Zs1NdI1GREImoyEyZMEP/5165v374iAHH58uV19i8tLa2z7V//+pfo6uoqlpeXG7YlJCSIrVq1MjxPS0sTAYi+vr5ifn6+Yfu6detEAOLPP/9s2DZ79uw6mQCICoVCPHv2rGHbkSNHRADiBx98YNgWHx8vurq6illZWYZtZ86cEZ2cnOq8Z30SEhJENze3G75eUVEh+vv7i506dRLLysoM23/55RcRgDhr1ixRFEXx6tWrIgDxnXfeueF7/fjjjyIAcf/+/TfNdb3CwkJRqVSKU6dONdq+YMECURAE8fz586IoiuKiRYtEAOLly5dNen9RFMVWrVqJAOp9JCUlGfZLSEgQAYgvvviiYZterxeHDBkiKhQKw+deu3atCEB86623jD7PI488IgqCYPi+njlzRpTJZOJDDz0k6nQ6o331en2dfDt37jRsy83Nrff3hcgacFqKyAoolUqMGzeuzvbr/8deVFSEvLw83HXXXSgtLcWpU6du+r4jRoyAt7e34fldd90FAEhNTb3psf3790d4eLjheZcuXeDp6Wk4VqfT4bfffsOwYcOMFr1GREQgLi7upu/fGAcOHEBubi5eeOEFqFQqw/YhQ4agXbt2WL9+PYDq3yeFQoHt27fj6tWr9b5X7QjPL7/8gsrKykZn8PT0RFxcHFatWmU0pbNy5Ur06tULLVu2NHr/devW3dLi7piYGGzZsqXOY+TIkXX2nThxouHj2inEiooK/PbbbwCADRs2QC6XY9KkSUbHTZ06FaIoYuPGjQCAtWvXQq/XY9asWZDJjH8c/HNasUOHDoY/PwDQrFkzREZGNurPElFTY7khsgLNmzeHQqGos/348eN46KGHoFar4enpiWbNmhkWnRYWFt70fWt/8NaqLTo3KgANHVt7fO2xubm5KCsrQ0RERJ396tt2K86fPw8AiIyMrPNau3btDK8rlUq8/fbb2LhxIwICAnD33XdjwYIFyM7ONuzft29fDB8+HHPmzIGfnx8efPBBfP7559BqtTfNMWLECGRmZmLPnj0AgHPnzuHgwYMYMWKE0T59+vTB008/jYCAADz++ONYtWpVo4uOn58f+vfvX+fRqlUro/1kMlmdNUBt27YFAMP6ovPnzyM4OBgeHh5G+9WelVf7+3bu3DnIZDJ06NDhpvlu9ueByJqw3BBZgetHaGoVFBSgb9++OHLkCObOnYuff/4ZW7Zswdtvvw0AjfqhKZfL690u1rOo1JzHSmHy5MlISUlBUlISVCoVZs6cifbt2+Pw4cMAqkci1qxZgz179mDixInIysrC+PHj0a1bNxQXFzf43vHx8XB1dcWqVasAAKtWrYJMJsOjjz5q2MfFxQU7d+7Eb7/9hieffBJHjx7FiBEjcP/999dZAG6LbO3PAzk2lhsiK7V9+3ZcuXIFX3zxBV566SUMHToU/fv3N5pmkpK/vz9UKhXOnj1b57X6tt2K2lGL06dP13nt9OnTdUY1wsPDMXXqVPz666/4+++/UVFRgffee89on169emHevHk4cOAAvv76axw/fhzfffddgznc3NwwdOhQrF69Gnq9HitXrsRdd91V5xo0MpkM/fr1w8KFC3HixAnMmzcP27Ztw++//34rX3699Hp9namglJQUADCcMdeqVStcvHgRRUVFRvvVTmXW/r6Fh4dDr9fjxIkTZstHZA1YboisVO3/lK//n3FFRQU+/PBDqSIZkcvl6N+/P9auXYuLFy8atp89e9awpuN2de/eHf7+/li+fLnR9NHGjRtx8uRJDBkyBED1dYHKy8uNjg0PD4eHh4fhuKtXr9YZZejatSsANHpq6uLFi/j0009x5MgRoykpAMjPz69zjCnvb4olS5YYPhZFEUuWLIGzszP69esHABg8eDB0Op3RfgCwaNEiCIJgWBM1bNgwyGQyzJ07t85IIEdkyJbxVHAiK9W7d294e3sjISEBkyZNgiAI+N///mdVP3TeeOMN/Prrr+jTpw+ef/55ww/UTp06ITk5uVHvUVlZibfeeqvOdh8fH7zwwgt4++23MW7cOPTt2xcjR440nAoeGhqKKVOmAKgeuejXrx8ee+wxdOjQAU5OTvjxxx+Rk5ODxx9/HADw5Zdf4sMPP8RDDz2E8PBwFBUV4ZNPPoGnpycGDx5805yDBw+Gh4cHpk2bBrlcjuHDhxu9PnfuXOzcuRNDhgxBq1atkJubiw8//BAtWrTAnXfeedP3z8rKwldffVVnu7u7O4YNG2Z4rlKpsGnTJiQkJCAmJgYbN27E+vXr8eqrrxpOwY+Pj8e9996L1157Denp6YiKisKvv/6KdevWYfLkyYaF4hEREXjttdfw5ptv4q677sLDDz8MpVKJ/fv3Izg4GElJSTfNTWSVpDpNi8gR3ehU8I4dO9a7/+7du8VevXqJLi4uYnBwsPjKK6+ImzdvFgGIv//+u2G/G50KXt+p0QDE2bNnG57f6FTwCRMm1Dm2VatWYkJCgtG2rVu3itHR0aJCoRDDw8PFTz/9VJw6daqoUqlu8LtwTe2pzfU9wsPDDfutXLlSjI6OFpVKpejj4yOOHj1avHDhguH1vLw8ccKECWK7du1ENzc3Ua1WizExMeKqVasM+xw6dEgcOXKk2LJlS1GpVIr+/v7i0KFDxQMHDtw0Z63Ro0eLAMT+/fvXeW3r1q3igw8+KAYHB4sKhUIMDg4WR44cKaakpNz0fRs6Ffz672vtqfPnzp0TBwwYILq6uooBAQHi7Nmz65zKXVRUJE6ZMkUMDg4WnZ2dxTZt2ojvvPOO0SnetT777DPD76+3t7fYt29fccuWLUb5hgwZUue4vn37in379r3p10fU1ARRtKL/BhKRXRg2bBiOHz+OM2fOSB3FrowdOxZr1qy56QJoIkfHNTdEdFvKysqMnp85cwYbNmzAPffcI00gInJ4XHNDRLeldevWGDt2LFq3bo3z589j2bJlUCgUeOWVV6SORkQOiuWGiG7LoEGD8O233yI7OxtKpRKxsbGYP38+2rRpI3U0InJQXHNDREREdoVrboiIiMiusNwQERGRXXG4NTd6vR4XL16Eh4dHnbveEhERkXUSRRFFRUUIDg6ucxf7f3K4cnPx4kWEhIRIHYOIiIhuQWZmJlq0aNHgPg5Xbjw8PABU/+Z4enpKnIaIiIgaQ6PRICQkxPBzvCEOV25qp6I8PT1ZboiIiGxMY5aUcEExERER2RWWGyIiIrIrLDdERERkV1huiIiIyK6w3BAREZFdYbkhIiIiu8JyQ0RERHaF5YaIiIjsCssNERER2RWWGyIiIrIrLDdERERkV1huiIiIyK6w3JhRUXkl/s4qlDoGERGRQ2O5MZO/swrRde4WjP18H0RRlDoOERGRw2K5MZM2Ae5wlgvIK67AmdxiqeMQERE5LJYbM1E6ydG9lQ8AYM+5KxKnISIiclwsN2YUG+4LAPjzXJ7ESYiIiBwXy40Z1Zabv9Lyoddz3Q0REZEUWG7MqHNzNdwUchSUVuJktkbqOERERA6J5caMnOUy9AjjuhsiIiIpSVpudu7cifj4eAQHB0MQBKxdu/amx3z99deIioqCq6srgoKCMH78eFy5Yj1FIrZ19dQUyw0REZE0JC03JSUliIqKwtKlSxu1/+7duzFmzBg89dRTOH78OFavXo19+/bhmWeesXDSxusd7gcA2JeWjyqdXuI0REREjsdJyk8eFxeHuLi4Ru+/Z88ehIaGYtKkSQCAsLAw/Otf/8Lbb79tqYgm6xDsCU+VEzTlVTh+UYOoEC+pIxERETkUm1pzExsbi8zMTGzYsAGiKCInJwdr1qzB4MGDpY5mIJcJ6BlWMzWVyqkpIiKipmZT5aZPnz74+uuvMWLECCgUCgQGBkKtVjc4raXVaqHRaIwelnbtejcsN0RERE3NpsrNiRMn8NJLL2HWrFk4ePAgNm3ahPT0dDz33HM3PCYpKQlqtdrwCAkJsXjO3jXl5kB6Piq57oaIiKhJCaKV3OVREAT8+OOPGDZs2A33efLJJ1FeXo7Vq1cbtu3atQt33XUXLl68iKCgoDrHaLVaaLVaw3ONRoOQkBAUFhbC09PTrF9DLb1eRLe3tuBqaSW+fz4W3Wpuy0BERES3RqPRQK1WN+rnt02N3JSWlkImM44sl8sB4IZ34lYqlfD09DR6WJpMJqAXTwknIiKShKTlpri4GMnJyUhOTgYApKWlITk5GRkZGQCAGTNmYMyYMYb94+Pj8cMPP2DZsmVITU3F7t27MWnSJPTs2RPBwcFSfAk3xHU3RERE0pD0VPADBw7g3nvvNTxPTEwEACQkJOCLL77ApUuXDEUHAMaOHYuioiIsWbIEU6dOhZeXF+677z6rOhW8Vu3F/A6evwptlQ5KJ7nEiYiIiByD1ay5aSqmzNndDlEU0WPeVuQVa/Hds70M01RERERkOrtdc2NLBEEwTE1x3Q0REVHTYbmxIN5nioiIqOmx3FhQ7cjN4cyrKKvQSZyGiIjIMbDcWFCoryuC1CpU6kQcPH9V6jhEREQOgeXGggRBuDY1lZoncRoiIiLHwHJjYb14vRsiIqImxXJjYbUjN0cvFKJYWyVxGiIiIvvHcmNhIT6uCPFxgU4vYn96vtRxiIiI7B7LTROoHb3Zy6kpIiIii2O5aQK8zxQREVHTYblpArGt/QAAxy8WorCsUuI0RERE9o3lpgkEqlVo7ecGvQjsS+O6GyIiIktiuWkivXifKSIioibBctNEahcV/3mOF/MjIiKyJJabJtKrptycyi5CfkmFxGmIiIjsF8tNE2nmoUTbAHcAwF+pnJoiIiKyFJabJnTtPlMsN0RERJbCctOEYrmomIiIyOJYbppQTJgvBAE4k1uM3KJyqeMQERHZJZabJuTtpkD7QE8AwN5UXu+GiIjIElhumhinpoiIiCyL5aaJGW6iyUXFREREFsFy08R6tvaBTADS8kpwqbBM6jhERER2h+WmiXmqnNG5uRoAp6aIiIgsgeVGArzPFBERkeWw3EiAF/MjIiKyHJYbCfQI9YGTTMCFq2XIzC+VOg4REZFdYbmRgJvSCVEhXgA4NUVERGRuLDcS4dQUERGRZbDcSOT6i/mJoihxGiIiIvvBciORbq28oZDLkK0pR1peidRxiIiI7AbLjURUznJEt/QCwKkpIiIic2K5kRDvM0VERGR+LDcSuv4+U1x3Q0REZB6SlpudO3ciPj4ewcHBEAQBa9euvekxWq0Wr732Glq1agWlUonQ0FB89tlnlg9rAV1bekHpJENecQXO5BZLHYeIiMguOEn5yUtKShAVFYXx48fj4YcfbtQxjz32GHJycrBixQpERETg0qVL0Ov1Fk5qGUonOXqE+mDX2TzsOXcFbQM8pI5ERERk8yQtN3FxcYiLi2v0/ps2bcKOHTuQmpoKHx8fAEBoaKiF0jWN2HBfQ7lJ6B0qdRwiIiKbZ1Nrbn766Sd0794dCxYsQPPmzdG2bVtMmzYNZWVlUke7Zb1q192kXYFez3U3REREt0vSkRtTpaamYteuXVCpVPjxxx+Rl5eHF154AVeuXMHnn39e7zFarRZardbwXKPRNFXcRunSQg1XhRwFpZU4ma1Bx2C11JGIiIhsmk2N3Oj1egiCgK+//ho9e/bE4MGDsXDhQnz55Zc3HL1JSkqCWq02PEJCQpo4dcOc5TL0DKueYuMp4URERLfPpspNUFAQmjdvDrX62uhG+/btIYoiLly4UO8xM2bMQGFhoeGRmZnZVHEb7fpTwomIiOj22FS56dOnDy5evIji4munTaekpEAmk6FFixb1HqNUKuHp6Wn0sDa1F/P7KzUfVTrbPPOLiIjIWkhaboqLi5GcnIzk5GQAQFpaGpKTk5GRkQGgetRlzJgxhv1HjRoFX19fjBs3DidOnMDOnTvx8ssvY/z48XBxcZHiSzCLjsFqeKicUKStwvGL1rUmiIiIyNZIWm4OHDiA6OhoREdHAwASExMRHR2NWbNmAQAuXbpkKDoA4O7uji1btqCgoADdu3fH6NGjER8fj/fff1+S/OYilwmICau5FQOnpoiIiG6LIDrYdf81Gg3UajUKCwutaopqxa40vPnLCfRt2wxfju8pdRwiIiKrYsrPb5tac2PPahcV70/PRyXX3RAREd0ylhsr0S7QA96uziit0OHohQKp4xAREdkslhsrIZMJhqsV83o3REREt47lxorUnhLORcVERES3juXGitSuuzmQfhXaKp3EaYiIiGwTy40VifB3h5+7EtoqPQ5nFEgdh4iIyCax3FgRQRCuTU1x3Q0REdEtYbmxMrVTU1x3Q0REdGtYbqxM7chNckYByiq47oaIiMhULDdWJtTXFYGeKlTo9Dh4/qrUcYiIiGwOy42VEQQBvQ2nhOdJnIaIiMj2sNxYoV5cVExERHTLWG6sUO2i4qMXClGsrZI4DRERkW1hubFCIT6uaOHtgiq9iP3p+VLHISIisiksN1aqdt3NXk5NERERmYTlxkrxPlNERES3huXGSsW29gMA/J1ViMKySonTEBER2Q6WGysVqFYhzM8NehHYl8Z1N0RERI3FcmPFeJ8pIiIi07HcWDHeZ4qIiMh0LDdWrFdNuTl5SYOrJRUSpyEiIrINLDdWrJmHEm383QEAf6Vx9IaIiKgxWG6sXO31bv7kuhsiIqJGYbmxclxUTEREZBqWGysXE+YLQQDO5BbjcpFW6jhERERWj+XGynm7KdAu0BMAsJdnTREREd0Uy40N4LobIiKixmO5sQG117vhyA0REdHNsdzYgJ6tfSATgLS8EmQXlksdh4iIyKqx3NgAT5UzOjVXAwD2pOZJnIaIiMi6sdzYiNpTwv88y6kpIiKihrDc2AjeZ4qIiKhxWG5sRI9QHzjJBFy4WobM/FKp4xAREVktlhsb4aZ0QpcWtetuOHpDRER0Iyw3NqR3uB8A3oqBiIioIZKWm507dyI+Ph7BwcEQBAFr165t9LG7d++Gk5MTunbtarF81ub6+0yJoihxGiIiIuskabkpKSlBVFQUli5datJxBQUFGDNmDPr162ehZNapWytvKOQyZGvKkX6F626IiIjq4yTlJ4+Li0NcXJzJxz333HMYNWoU5HK5SaM9tk7lLEfXll7Yl5aPPeeuIMzPTepIREREVsfm1tx8/vnnSE1NxezZsxu1v1arhUajMXrYsmv3meLF/IiIiOpjU+XmzJkzmD59Or766is4OTVu0CkpKQlqtdrwCAkJsXBKy7p2n6l8rrshIiKqh82UG51Oh1GjRmHOnDlo27Zto4+bMWMGCgsLDY/MzEwLprS8ri29oHSSIa9Yi7O5xVLHISIisjqSrrkxRVFREQ4cOIDDhw9j4sSJAAC9Xg9RFOHk5IRff/0V9913X53jlEollEplU8e1GKWTHN1DvbH77BXsSb2CNgEeUkciIiKyKjZTbjw9PXHs2DGjbR9++CG2bduGNWvWICwsTKJkTa93uB92n72CP89ewZjYUKnjEBERWRVJy01xcTHOnj1reJ6Wlobk5GT4+PigZcuWmDFjBrKysvDf//4XMpkMnTp1Mjre398fKpWqznZ716t23U3aFej1ImQyQeJERERE1kPSNTcHDhxAdHQ0oqOjAQCJiYmIjo7GrFmzAACXLl1CRkaGlBGtUpcWargq5CgorcSp7CKp4xAREVkVQXSwU240Gg3UajUKCwvh6ekpdZxblvDZPuxIuYyZQzvgqTsdZ0qOiIgckyk/v23mbCky1ttwKwZe74aIiOh6LDc2qvY+U3+l5UOnd6jBNyIiogax3NiojsFqeKicUFReheMXC6WOQ0REZDVYbmyUXCYgJswHQPVdwomIiKgay40Niw33AwD8yXJDRERkwHJjw2rvM7U/PR+VOr3EaYiIiKwDy40NaxfoAW9XZ5RW6HD0AtfdEBERASw3Nk0mExATVnuXcE5NERERASw3Nq/2lPA/eb0bIiIiACw3Nq/2Yn4H0q9CW6WTOA0REZH0WG5sXIS/O/zcldBW6ZGcUSB1HCIiIsmx3Ng4QRDQq3XN9W647oaIiIjlxh5cW3fDckNERMRyYwd611zMLzmjAOWVXHdDRESOjeXGDoT6uiLQU4UKnR4Hz1+VOg4REZGkWG7sgCAIhqkp3meKiIgcHcuNnai9FQOvd0NERI7O5HLz5ZdfYv369Ybnr7zyCry8vNC7d2+cP3/erOGo8WpHbo5eKESJtkriNERERNIxudzMnz8fLi4uAIA9e/Zg6dKlWLBgAfz8/DBlyhSzB6TGCfFxRQtvF1TpRexPz5c6DhERkWRMLjeZmZmIiIgAAKxduxbDhw/Hs88+i6SkJPzxxx9mD0iNVzs1xevdEBGRIzO53Li7u+PKleofnr/++ivuv/9+AIBKpUJZWZl505FJuKiYiIgIcDL1gPvvvx9PP/00oqOjkZKSgsGDBwMAjh8/jtDQUHPnIxPUlpu/swqhKa+Ep8pZ4kRERERNz+SRm6VLlyI2NhaXL1/G999/D1/f6h+oBw8exMiRI80ekBovSO2CMD836EVgXyrX3RARkWMyeeTGy8sLS5YsqbN9zpw5ZglEt6dXa1+k5ZVgT+oV9O8QIHUcIiKiJmfyyM2mTZuwa9cuw/OlS5eia9euGDVqFK5e5dVxpcZ1N0RE5OhMLjcvv/wyNBoNAODYsWOYOnUqBg8ejLS0NCQmJpo9IJmm9oypE5c0uFpSIXEaIiKipmdyuUlLS0OHDh0AAN9//z2GDh2K+fPnY+nSpdi4caPZA5Jpmnko0cbfHQDwVxpHb4iIyPGYXG4UCgVKS0sBAL/99hsGDBgAAPDx8TGM6JC0ODVFRESOzORyc+eddyIxMRFvvvkm9u3bhyFDhgAAUlJS0KJFC7MHJNPxYn5EROTITC43S5YsgZOTE9asWYNly5ahefPmAICNGzdi0KBBZg9IputVU25ScopxuUgrcRoiIqKmJYiiKEodoilpNBqo1WoUFhbC09NT6jgWE/efP3DykgYfjIxGfFSw1HGIiIhuiyk/v02+zg0A6HQ6rF27FidPngQAdOzYEQ888ADkcvmtvB1ZQGxrX5y8pMGe1CssN0RE5FBMLjdnz57F4MGDkZWVhcjISABAUlISQkJCsH79eoSHh5s9JJkuNtwXn+1Ow14uKiYiIgdj8pqbSZMmITw8HJmZmTh06BAOHTqEjIwMhIWFYdKkSZbISLegZ5gPZAKQmleC7MJyqeMQERE1GZPLzY4dO7BgwQL4+PgYtvn6+uL//u//sGPHDpPea+fOnYiPj0dwcDAEQcDatWsb3P+HH37A/fffj2bNmsHT0xOxsbHYvHmzqV+CQ1C7OKNTczUAYE9qnsRpiIiImo7J5UapVKKoqKjO9uLiYigUCpPeq6SkBFFRUVi6dGmj9t+5cyfuv/9+bNiwAQcPHsS9996L+Ph4HD582KTP6ygMp4RzaoqIiByIyWtuhg4dimeffRYrVqxAz549AQB//fUXnnvuOTzwwAMmvVdcXBzi4uIavf/ixYuNns+fPx/r1q3Dzz//jOjoaJM+tyPoFe6Lj3am8no3RETkUEweuXn//fcRHh6O2NhYqFQqqFQq9OnTBxEREXXKh6Xp9XoUFRUZTZH9k1arhUajMXo4ih6hPnCSCcjML0NmfqnUcYiIiJqEySM3Xl5eWLduHc6ePWs4Fbx9+/aIiIgwe7ibeffdd1FcXIzHHnvshvskJSVhzpw5TZjKergrndClhRqHMgqwJ/UKQnxcpY5ERERkcSaP3NSKiIhAfHw84uPjERERgaNHj5q85uZ2fPPNN5gzZw5WrVoFf3//G+43Y8YMFBYWGh6ZmZlNltEa1N5niqeEExGRo7jlcvNPoihCp9OZ6+0a9N133+Hpp5/GqlWr0L9//wb3VSqV8PT0NHo4ktjWfgCq7zPlYBejJiIiB2W2ctNUvv32W4wbNw7ffvut4aaddGPdWnlDIZfhUmE50q9w3Q0REdk/SctNcXExkpOTkZycDABIS0tDcnIyMjIyAFRPKY0ZM8aw/zfffIMxY8bgvffeQ0xMDLKzs5GdnY3CwkIp4tsEF4UcXVt6AeAp4URE5BgaXW7+ecbRPx/1XfvmZg4cOIDo6GjDadyJiYmIjo7GrFmzAACXLl0yFB0A+Pjjj1FVVYUJEyYgKCjI8HjppZdM/tyOxHC9G54STkREDqDRdwWXyWQQBOGGr4uiCEEQmmzdza1ylLuCX29v6hU8/vFe+Lkrsf+1fg1+H4mIiKyRRe4K/vvvv992MJJGdEsvKJ1kyCvW4mxuMdoEeEgdiYiIyGIaXW769u1ryRxkQUonObqHemP32SvYk3qF5YaIiOyazZ0tRbeG95kiIiJHwXLjIAwX80u9Ar2e17shIiL7xXLjILq08IKrQo6rpZU4lW36mW1ERES2guXGQTjLZegRWn2DUZ4STkRE9ozlxoHUTk1x3Q0REdkzk+8K/tBDD9V7nRRBEKBSqRAREYFRo0YhMjLSLAHJfGoXFf+VdgU6vQi5jNe7ISIi+2PyyI1arca2bdtw6NAhCIIAQRBw+PBhbNu2DVVVVVi5ciWioqKwe/duS+Sl29Ax2BMeKicUlVfh+EXesoKIiOyTyeUmMDAQo0aNQmpqKr7//nt8//33OHfuHJ544gmEh4fj5MmTSEhIwL///W9L5KXb4CSXISasZt0Np6aIiMhOmVxuVqxYgcmTJ0Mmu3aoTCbDiy++iI8//hiCIGDixIn4+++/zRqUzKMX7zNFRER2zuRyU1VVhVOnTtXZfurUKcN9pVQqFe9fZKVqFxXvT8tHpU4vcRoiIiLzM3lB8ZNPPomnnnoKr776Knr06AEA2L9/P+bPn48xY8YAAHbs2IGOHTuaNymZRftAT3i5OqOgtBJHLxSiWytvqSMRERGZlcnlZtGiRQgICMCCBQuQk5MDAAgICMCUKVMM62wGDBiAQYMGmTcpmYVMJqBXmC82Hc/G3tQrLDdERGR3BFEUb/la/BqNBgBueutxa2LKLdPt1Zd/pmP2T8dxZ4Qfvno6Ruo4REREN2XKz2+TR26u56jlwNbVrrs5cD4f2iodlE5yiRMRERGZj8kLinNycvDkk08iODgYTk5OkMvlRg+yfm383eHnrkB5pR7JGQVSxyEiIjIrk0duxo4di4yMDMycORNBQUE8K8oGCYKAXq198cvRS9iTegUxNaeHExER2QOTy82uXbvwxx9/oGvXrhaIQ00lNrym3Jy7gsn9pU5DRERkPiZPS4WEhOA21iCTlai9z9ThjAKUV+okTkNERGQ+JpebxYsXY/r06UhPT7dAHGoqYX5uCPRUoUKnx8HzV6WOQ0REZDYmT0uNGDECpaWlCA8Ph6urK5ydnY1ez8/PN1s4shxBEBAb7osfD2dhz7kr6BPhJ3UkIiIiszC53CxevNgCMUgKsa1ryg3vM0VERHbE5HKTkJBgiRwkgdrr3RzJLECJtgpuytu67BEREZFVaNRPM41GY7hgX+1ViW+EF/azHSE+rmjh7YILV8uwPz0f90T6Sx2JiIjotjVqQbG3tzdyc3MBAF5eXvD29q7zqN1OtqX2rClOTRERkb1o1MjNtm3b4OPjAwD4/fffLRqImlZsuC9WH7yAvedYboiIyD40qtz07du33o/J9tWuuzmWVQhNeSU8Vc43OYKIiMi63dIK0oKCAuzbtw+5ubnQ6/VGr40ZM8YswahpBKldEObnhrS8EuxPy0e/9gFSRyIiIrotJpebn3/+GaNHj0ZxcTE8PT2N7i0lCALLjQ3q1doXaXkl+PPcFZYbIiKyeSZfoXjq1KkYP348iouLUVBQgKtXrxoevICfbaqdmtrDdTdERGQHTC43WVlZmDRpElxdXS2RhyTQq3X1YvGT2RoUlFZInIaIiOj2mFxuBg4ciAMHDlgiC0nE30OFNv7uEEVgbypH34iIyLaZvOZmyJAhePnll3HixAl07ty5zr2lHnjgAbOFo6YTG+6LM7nF2HMuD4M6BUodh4iI6JaZPHLzzDPPIDMzE3PnzsWjjz6KYcOGGR4PPfSQSe+1c+dOxMfHIzg4GIIgYO3atTc9Zvv27bjjjjugVCoRERGBL774wtQvgerBi/kREZG9MLnc6PX6Gz50Op1J71VSUoKoqCgsXbq0UfunpaVhyJAhuPfee5GcnIzJkyfj6aefxubNm039MugfYmrKTUpOMfKKtRKnISIiunWS3ikxLi4OcXFxjd5/+fLlCAsLw3vvvQcAaN++PXbt2oVFixZh4MCBlorpEHzcFGgf5ImTlzTYm3oFQ7sESx2JiIjoljSq3Lz//vt49tlnoVKp8P777ze476RJk8wSrD579uxB//79jbYNHDgQkydPvuExWq0WWu21kYib3fjTkcW29sXJSxr8eY7lhoiIbFejys2iRYswevRoqFQqLFq06Ib7CYJg0XKTnZ2NgADji8wFBARAo9GgrKwMLi4udY5JSkrCnDlzLJbJnsSG++Kz3Wm8zxQREdm0RpWbtLS0ej+2BTNmzEBiYqLhuUajQUhIiISJrFfPMB/IBCA1rwQ5mnIEeKqkjkRERGQykxcUSykwMBA5OTlG23JycuDp6VnvqA0AKJVKeHp6Gj2ofmoXZ3RqrgbAqxUTEZHtuqUFxRcuXMBPP/2EjIwMVFQYX9F24cKFZglWn9jYWGzYsMFo25YtWxAbG2uxz+loYlv74uiFQvx5Lg/DoptLHYeIiMhkJpebrVu34oEHHkDr1q1x6tQpdOrUCenp6RBFEXfccYdJ71VcXIyzZ88anqelpSE5ORk+Pj5o2bIlZsyYgaysLPz3v/8FADz33HNYsmQJXnnlFYwfPx7btm3DqlWrsH79elO/DLqBXuG++GhnKq93Q0RENsvkaakZM2Zg2rRpOHbsGFQqFb7//ntkZmaib9++ePTRR016rwMHDiA6OhrR0dEAgMTERERHR2PWrFkAgEuXLiEjI8Owf1hYGNavX48tW7YgKioK7733Hj799FOeBm5GPUJ94CwXkJlfhl+PZ0sdh4iIyGSCKIqiKQd4eHggOTkZ4eHh8Pb2xq5du9CxY0ccOXIEDz74INLT0y0U1Tw0Gg3UajUKCwu5/uYGkjaexEc7UuHnrsCvU/rCx00hdSQiInJwpvz8Nnnkxs3NzbDOJigoCOfOnTO8lpeXZ+rbkRWa0r8t2ga4I6+4Aq+vPQYT+y8REZGkTC43vXr1wq5duwAAgwcPxtSpUzFv3jyMHz8evXr1MntAanoqZznee7QrnGQCNhzLxs9HL0kdiYiIqNFMLjcLFy5ETEwMAGDOnDno168fVq5cidDQUKxYscLsAUkanVuoMfG+CADArHV/I1dTLnEiIiKixjFpzY1Op8Pu3bvRpUsXeHl5WTCW5XDNTeNV6vR46MPd+DtLg37t/PFpQncIgiB1LCIickAWW3Mjl8sxYMAAXL169bYCkm1wlsuw8LGuUMhl2HoqF6sPXpA6EhER0U2ZPC3VqVMnpKamWiILWaG2AR6YOqAtAGDuzydw4WqpxImIiIgaZnK5eeuttzBt2jT88ssvuHTpEjQajdGD7M/Td7VGt1beKNZW4ZU1R6HX8+wpIiKyXo0uN3PnzkVJSQkGDx6MI0eO4IEHHkCLFi3g7e0Nb29veHl5wdvb25JZSSJymYB3H42CylmGP89dwVd/nZc6EhER0Q01ekGxXC7HpUuXcPLkyQb369u3r1mCWQoXFN+6L/9Mx+yfjsPFWY4NL92FMD83qSMREZGDMOXnd6PvLVXbgay9vJDlPNmrFTYfz8af565g2uojWPWvWMhlPHuKiIisi0lrbngasGOTyQQseKQL3JVOOHj+Klbs4sJyIiKyPibdFbxt27Y3LTj5+fm3FYisWwtvV8wa2gGvfH8U725OwT2R/mgb4CF1LCIiIgOTys2cOXOgVqstlYVsxKPdW2DT8WxsO5WLqauO4IcXesNZbvKJd0RERBbR6AXFMpkM2dnZ8Pf3t3Qmi+KCYvPI1ZTj/kU7UVhWiSn92+Kl/m2kjkRERHbMIlco5nobup6/pwpvDusEAPhg2xn8nVUocSIiIqJqjS43JtyCihxEfJcgDO4ciCq9iMRVydBW6aSORERE1Phyo9frbX5KisxLEAS8+WAn+LkrkJJTjEVbzkgdiYiIyPTbLxBdz9ddiXkPdQYAfLzzHA6e501ViYhIWiw3dNsGdgzEw3c0h14Epq0+grIKTk8REZF0WG7ILGbHd0SgpwppeSV4e9MpqeMQEZEDY7khs1C7OGPBI10AAF/8mY4/z+VJnIiIiBwVyw2Zzd1tm2F0TEsAwMurj6KovFLiRERE5IhYbsisXh3cHiE+LsgqKMO89Q3fQZ6IiMgSWG7IrNyUTnj3kSgIAvDd/kz8fipX6khERORgWG7I7GJa+2J8nzAAwL+/P4qC0gqJExERkSNhuSGLeHlgJFo3c0NukRazfzoudRwiInIgLDdkESpnORY+1hUyAViXfBEbj12SOhIRETkIlhuymK4hXnjhnggAwGtr/0ZesVbiRERE5AhYbsiiJvVrg3aBHsgvqcCrPxzjDViJiMjiWG7IohROMix8rCuc5QJ+PZGDtclZUkciIiI7x3JDFtch2BOT+7cFAMxadxyXCsskTkRERPaM5YaaxL/ubo2oEC8UlVfh399zeoqIiCyH5YaahJNchvcejYLSSYadKZfx7b5MqSMREZGdYrmhJhPh746XB0YCAN5afwIZV0olTkRERPbIKsrN0qVLERoaCpVKhZiYGOzbt6/B/RcvXozIyEi4uLggJCQEU6ZMQXl5eROlpdsxvk8Yeob5oLRCh5fXHIFez+kpIiIyL8nLzcqVK5GYmIjZs2fj0KFDiIqKwsCBA5GbW/89ib755htMnz4ds2fPxsmTJ7FixQqsXLkSr776ahMnp1shkwl495EouCrk+CstH1/8mS51JCIisjOSl5uFCxfimWeewbhx49ChQwcsX74crq6u+Oyzz+rd/88//0SfPn0watQohIaGYsCAARg5cuRNR3vIerT0dcVrQ9oDAN7edArnLhdLnIiIiOyJpOWmoqICBw8eRP/+/Q3bZDIZ+vfvjz179tR7TO/evXHw4EFDmUlNTcWGDRswePDgevfXarXQaDRGD5LeqJ4tcVcbP2ir9Ji66giqdHqpIxERkZ2QtNzk5eVBp9MhICDAaHtAQACys7PrPWbUqFGYO3cu7rzzTjg7OyM8PBz33HPPDaelkpKSoFarDY+QkBCzfx1kOkEQsOCRLvBQOSE5swAf7UyVOhIREdkJyaelTLV9+3bMnz8fH374IQ4dOoQffvgB69evx5tvvlnv/jNmzEBhYaHhkZnJU5CtRZDaBW/EdwQALP4tBScvcVSNiIhun5OUn9zPzw9yuRw5OTlG23NychAYGFjvMTNnzsSTTz6Jp59+GgDQuXNnlJSU4Nlnn8Vrr70Gmcy4rymVSiiVSst8AXTbHr6jOTYdz8aWEzlIXHUE6yb0gcLJ5jo3ERFZEUl/iigUCnTr1g1bt241bNPr9di6dStiY2PrPaa0tLROgZHL5QDAq97aIEEQMP+hzvB2dcbJSxp8sO2M1JGIiMjGSf5f5MTERHzyySf48ssvcfLkSTz//PMoKSnBuHHjAABjxozBjBkzDPvHx8dj2bJl+O6775CWloYtW7Zg5syZiI+PN5Qcsi3NPJSY91BnAMCH28/hSGaBtIGIiMimSTotBQAjRozA5cuXMWvWLGRnZ6Nr167YtGmTYZFxRkaG0UjN66+/DkEQ8PrrryMrKwvNmjVDfHw85s2bJ9WXQGYwuHMQHogKxk9HLmLq6iP45cU7oXJmWSUiItMJooPN5Wg0GqjVahQWFsLT01PqOHSdgtIK3L9oJy4XafHMXWF4bUgHqSMREZGVMOXnt+TTUkS1vFwVeHt49fTUp7vSsC8tX+JERERki1huyKrc1y4AI7qHQBSBaauPoERbJXUkIiKyMSw3ZHVeH9oezb1ckJFfiqSNJ6WOQ0RENoblhqyOh8oZCx7pAgD4am8GdqZcljgRERHZEpYbskp9IvyQENsKAPDv74+isKxS4kRERGQrWG7Iav07rh1CfV1xqbAcc38+IXUcIiKyESw3ZLVcFU5477EoyATg+0MXsOVEzs0PIiIih8dyQ1atWysfPHN3awDAjB+OIb+kQuJERERk7VhuyOpN6d8WbQPckVesxcx1f0sdh4iIrBzLDVk9lbMc7z3aFU4yAeuPXsLPRy5KHYmIiKwYyw3ZhM4t1JhwbwQAYOa6v5GrKZc4ERERWSuWG7IZE++LQMdgTxSUVmLGD8fgYLdFIyKiRmK5IZvhLJdh4WNdoZDLsPVULlYfvCB1JCIiskIsN2RTIgM9kDigLQBg7s8nkFVQJnEiIiKyNiw3ZHOeuas17mjphWJtFf695ij0ek5PERHRNSw3ZHPkMgHvPdYVKmcZdp3Nw9d/nZc6EhERWRGWG7JJYX5umBHXHgAwf8MppOeVSJyIiIisBcsN2awne7VC73BflFXqMG31Eeg4PUVERGC5IRsmkwlY8EgXuCudcOD8VazYlSp1JCIisgIsN2TTWni7YubQ6umpdzenICWnSOJEREQkNZYbsnmPdQ/BvZHNUKHTY+qqI6jU6aWOREREEmK5IZsnCAL+b3gXqF2ccSyrEB/+fk7qSEREJCGWG7ILAZ4qzH2wIwDgg21n8HdWocSJiIhIKiw3ZDceiArG4M6BqNKLmLrqCLRVOqkjERGRBFhuyG4IgoA3H+wEP3cFTucUYfFvZ6SOREREEmC5Ibvi667EvIc6AwA+2nEOB89flTgRERE1NZYbsjsDOwbi4ejm0ItA4qpknLykkToSERE1IZYbskuzH+iIILUK56+UYvD7f+Dfa44iV1MudSwiImoCLDdkl9QuzljzfG8M7RIEUQRWHsjEPe9uxwdbz6CsgguNiYjsmSCKokPdkEej0UCtVqOwsBCenp5Sx6EmcPB8Pt785SSSMwsAAEFqFV4eGIlhXZtDJhOkDUdERI1iys9vlhtyCKIo4uejl/D2xlPIKigDAHRursbrQ9ojprWvxOmIiOhmWG4awHLj2Mordfh8dzqW/n4WxdoqAMCgjoGYHtcOoX5uEqcjIqIbYblpAMsNAUBesRaLtqTg230Z0IuAs1zAmNhQTLqvDdSuzlLHIyKif2C5aQDLDV0vJacI89afxI6UywAAL1dnvNSvDZ7o1QrOcq63JyKyFqb8/LaKf72XLl2K0NBQqFQqxMTEYN++fQ3uX1BQgAkTJiAoKAhKpRJt27bFhg0bmigt2ZO2AR74cnxPfDm+J9oGuKOgtBJzfj6BgYt24tfj2XCw7k9EZBckLzcrV65EYmIiZs+ejUOHDiEqKgoDBw5Ebm5uvftXVFTg/vvvR3p6OtasWYPTp0/jk08+QfPmzZs4OdmTvm2bYcOkuzD/oc7wc1cgNa8Ez/7vIEZ+spc34SQisjGST0vFxMSgR48eWLJkCQBAr9cjJCQEL774IqZPn15n/+XLl+Odd97BqVOn4Oxs+toITkvRzRSVV2L5jnP45I80VFTpIQjA8DtaYNqASASqVVLHIyJySDYzLVVRUYGDBw+if//+hm0ymQz9+/fHnj176j3mp59+QmxsLCZMmICAgAB06tQJ8+fPh05X/4XZtFotNBqN0YOoIR4qZ7w8sB22Te2LB7sGQxSBNQcv4N53t2PRlhSUVlRJHZGIiBogabnJy8uDTqdDQECA0faAgABkZ2fXe0xqairWrFkDnU6HDRs2YObMmXjvvffw1ltv1bt/UlIS1Gq14RESEmL2r4PsUwtvV/zn8Wj8+EJvdGvljbJKHf6z9QzufXc7Vh/IhF7P9ThERNZI8jU3ptLr9fD398fHH3+Mbt26YcSIEXjttdewfPnyevefMWMGCgsLDY/MzMwmTky2LrqlN9Y8F4sPR9+BEB8X5Gi0eHnNUcQv2YU/z+VJHY+IiP7BScpP7ufnB7lcjpycHKPtOTk5CAwMrPeYoKAgODs7Qy6XG7a1b98e2dnZqKiogEKhMNpfqVRCqVSaPzw5FEEQMLhzEPq198eXf6bjg21ncfyiBqM++Qv3dwjAjLh2aN3MXeqYREQEiUduFAoFunXrhq1btxq26fV6bN26FbGxsfUe06dPH5w9exZ6vd6wLSUlBUFBQXWKDZG5KZ3kePbucOx4+V4kxLaCXCZgy4kcDFi0E2/8dBxXSyqkjkhE5PAkn5ZKTEzEJ598gi+//BInT57E888/j5KSEowbNw4AMGbMGMyYMcOw//PPP4/8/Hy89NJLSElJwfr16zF//nxMmDBBqi+BHJCPmwJzHuyEzZPvQr92/qjSi/jiz3T0fed3fPpHKiqq9Dd/EyIisghJp6UAYMSIEbh8+TJmzZqF7OxsdO3aFZs2bTIsMs7IyIBMdq2DhYSEYPPmzZgyZQq6dOmC5s2b46WXXsK///1vqb4EcmAR/h5YMbYHdp3Jw1vrT+BUdhHeWn8S/9t7HjPi2mFgx0AIAu88TkTUlCS/zk1T43VuyFJ0ehHfH7yAd349jctFWgBAz1AfvD60Pbq08JI2HBGRjeO9pRrAckOWVqKtwkc7zuHjP1JRXlk9PfVwdHNMGxiJYC8XidMREdkmlpsGsNxQU7lYUIZ3N5/GD4ezAABKJxmevbs1nusbDjel5DPCREQ2heWmASw31NSOXijAW+tPYl9aPgCgmYcS0wa0xSPdQiCXcT0OEVFjsNw0gOWGpCCKIjYfz0HSxpM4f6UUANAu0AOvD+mAO9v4SZyOiMj6sdw0gOWGpFRRpcf/9p7Hf35Lgaa8+h5V97Xzx6uD2yHC30PidERE1ovlpgEsN2QNrpZU4P1tZ/C/PedRpRchlwkYHdMSL/VrA193XlGbiOifWG4awHJD1iT1cjGSNp7ClhPVtyDxUDph4n0RGNsnFEon+U2OJiJyHCw3DWC5IWu059wVvLX+BI5f1AAAQnxcMH1QewzuzIsAEhEBLDcNYrkha6XXi/jhcBbe2XwKOZrqiwB2a+WN14e0R3RLb4nTERFJi+WmASw3ZO1KK6rw8c5UfLQjFWWVOgDAkM5BiI8KQp8IP3ionCVOSETU9FhuGsByQ7YiR1OOdzefxppDF1D7t9RJJqB7qDfujfTHPZH+aBvgzmkrInIILDcNYLkhW3P8YiHWHLyA7acvIy2vxOi1YLUKfSP9cU9kM/SJ8IM7r3xMRHaK5aYBLDdky85fKcH205fx++lc7Dl3BdoqveE1Z7mAHqE+uCeyGe6N9EeEP0d1iMh+sNw0gOWG7EV5pQ57Uq9gR03Zqb3yca3mXi64J7IZ7on0R+9wX97PiohsGstNA1huyF6l5ZVg++lc/H76MvamXkHFdaM6CrkMPcN8DGUnvJkbR3WIyKaw3DSA5YYcQVmFDntS8wxTWJn5ZUavt/B2MUxfxYb7wlXBUR0ism4sNw1guSFHI4oiUvOq1+psP52Lv1LzUaG7blTHSYaYMB/cE+mPeyObIcyPozpEZH1YbhrAckOOrrSiCnvOXcHvp3Px+6nLyCowHtVp6eNqGNXp1doXLgreBoKIpMdy0wCWG6JrRFHEucvFhumrfWn5qNRd+ydB6SRDr9a+hrU6YX5uEqYlIkfGctMAlhuiGyvRVuHPmlGdHafrjuqE+rrinprr6vRq7QuVM0d1iKhpsNw0gOWGqHFEUcSZ3GJsP52L7acvY3963VGd2HDfmqslN0MrX47qEJHlsNw0gOWG6NYUa6uw+2yeoexcKiw3er21nxv61qzV6Rnmw1EdIjIrlpsGsNwQ3T5RFHE6p8hwBtaB9Kuo0l/7p8TFWV4zqlO9VifEx1XCtERkD1huGsByQ2R+ReWVNaM61QuTczRao9dbN3PDvZH+uLttM3QI8oSfu4KnmxORSVhuGsByQ2RZoijiVHYRfq+Zvjp4/ip0euN/ZrxdndE2wKPm4W742NtNIVFqIrJ2LDcNYLkhalqFZZWGtTp/peUjI78UN/pXx89dichAd7Tx90BkYHXxaRPgAU+Vc9OGJiKrw3LTAJYbImmVVehw7nIxTmcXISW3CGdyqj/+52nn1wtSq9AmwANt/d3RNrB6lKeNvztvBkrkQFhuGsByQ2SdirVVOJtbjJTsIqTkFCGl5uNsTfkNj2nh7VJddALcEVkztRXh784ztYjsEMtNA1huiGxLYVklzuYW4XR2MVJyinCm5uO8Ym29+wsC0MrHtXqk57r1PK2buUHpxNJDZKtYbhrAckNkH66WVFSP8OQUISWn2PDx1dLKeveXywSE+rpet5C5uvyE+rnBWS5r4vREZCqWmwaw3BDZL1EUkVdcgTM5RThdU3pqPy4qr6r3GGe5gNZ+7oaprdoRn1a+bpDLeLo6kbVguWkAyw2R4xFFETka7XUjPdeKT0mFrt5jlE4yhDdzr57aCvRAW//q0Z4W3i6QsfQQNTmWmwaw3BBRLVEUkVVQVn3GVk3pOZNTjDO5RSiv1Nd7jIuzHBH+7mju5YJAtQpBahWCvFwQpFYh0FOFAE8VFE6c5iIyN5abBrDcENHN6PQiLlwtNVrLk5JTjHO5xajQ1V96aglC9fV6asvOP8tPkNoFAWolFzcTmcjmys3SpUvxzjvvIDs7G1FRUfjggw/Qs2fPmx733XffYeTIkXjwwQexdu3aRn0ulhsiulVVOj3O55fiXG4xsjXluFhQjuzCMlwqLEe2phyXCstRUdVw+anl565AoFqFQE8XBHupDKNAtc8DPFU8pZ3oOqb8/Jb8ClgrV65EYmIili9fjpiYGCxevBgDBw7E6dOn4e/vf8Pj0tPTMW3aNNx1111NmJaIHJmTvHodTngz93pfF0UR+SUV1WWnsByXaotPYXXxqX2urdIjr7gCecUV+DtLc8PP5+OmQKCn6rry41I9+uN17WMXBQsQ0T9JPnITExODHj16YMmSJQAAvV6PkJAQvPjii5g+fXq9x+h0Otx9990YP348/vjjDxQUFHDkhohsgiiKKCitNCo715ef7MJyXCwsu+Gan3/ycnVGkLpm2kutQpDnddNgNaNBrgrJ/x9LdNtsZuSmoqICBw8exIwZMwzbZDIZ+vfvjz179tzwuLlz58Lf3x9PPfUU/vjjjwY/h1arhVZ77WJfGs2N/5dERGRpgiDA200BbzcFOgTX/w+0KIooLKusd9Tn+hGh0godCkorUVBaiZOXbvxvm9rF2ajsBKldaqbEVPB1V8DbVQEfNwWnwchuSFpu8vLyoNPpEBAQYLQ9ICAAp06dqveYXbt2YcWKFUhOTm7U50hKSsKcOXNuNyoRUZMRBAFergp4uSrQPujGBUhTXmVUdqrLz7WPLxWUoaRCh8KyShSWVeJUdlGDn9fFWQ5vV+fq8uWqqPnV2VB+vFyd4XPdaz6uCk6LkVWyqbHKoqIiPPnkk/jkk0/g5+fXqGNmzJiBxMREw3ONRoOQkBBLRSQiahKCIEDt4gy1izMiAz1uuF9ReWXNVNd1i59rnucUliO/tAJXSypQpRdRVqlDWaEOFwtvfD+vf1I6yWqKjwI+bs7Vv7oqDCXJ8JrrtXLkqpBDEHitILIcScuNn58f5HI5cnJyjLbn5OQgMDCwzv7nzp1Deno64uPjDdv0+up5aScnJ5w+fRrh4eFGxyiVSiiVSgukJyKyfh4qZ3ionNEm4MYFSBRFFGurcLWkEldLKwyF52ppZc2vNY/a10sqUFBaiQqdHtoqvWGkqLEUchm83apHhP45KlRbkv75mrvSiYWIGk3ScqNQKNCtWzds3boVw4YNA1BdVrZu3YqJEyfW2b9du3Y4duyY0bbXX38dRUVF+M9//sMRGSKiWyAIgqEEtfR1bdQxoiiipEJ3XfmpLkLVxaemIBnKUc1rpRWoqNKjQqdHjkaLHE39Nz+tj7NcuDYqVFt+rps283RxhqfKGZ4uTtW/1nzsrnSCE+8d5nAkn5ZKTExEQkICunfvjp49e2Lx4sUoKSnBuHHjAABjxoxB8+bNkZSUBJVKhU6dOhkd7+XlBQB1thMRkeUIggB3ZXV5CPFpfCEqq9QZRn7ya4tRSQXySyurS9F1rxWUVuBKSQW0VXpU6kRcLtLiclHjC1EtN4Ucni7O8FBVFx8PlZOhDNX3ce1+tUVJ6STjqJGNkbzcjBgxApcvX8asWbOQnZ2Nrl27YtOmTYZFxhkZGZDJ2LqJiGydIAhwVTjBVeGEFt6NP66sQmc0HZZfWmEoQrUjQ5rySmjKKlFUXlXzcRXKKqvvG1ZSoUNJhQ6XCm8tt0Iug6eLEzxUzvD8RwG6VoTq+dilen83hRPvR9bEJL/OTVPjdW6IiBxDpU5fXXauKz1FNcVHU14JTc1r1dtrP66q2acSRdoqmOMnpCAAHsqaclRTeDyMptCuFSY3ZXUZclHIr/2qlMPVufpjR75vmc1c54aIiMhSnOXVZ3L5uClu6Xi9XkRJRZWhBBmKkramINUUoBsXpCpU6PQQRVS/R3kVsgrKbvNrEuDiLIeb8h8FSCGvGRWTVz+UTnB1rvm1dpvCCW4KeU1hcjK8j6tCbndTbyw3RERE9ZDJri20bu7lckvvUV6pM0yTFV03WnRt+sz445IKHcoqdCipqKr+VVuF0godqvTVQ0iVOhGVuuqiZE4yAUblyOX6IlS7XfmPAmX4uPpXN6UcLs5O1SNNCic085DuTGWWGyIiIgtROcuhcpbD/8Zn4jdKRZXeUHpKK3QorfnVaJu2CqWVOpRqdUb7XPu17se1t/nQi0CxtgrFWvOUJh83BQ7NvN8s73UrWG6IiIisnMJJBoWTDGpXZ7O+r67m4o2lNSNEtSNG1xeh6tGkKpRodSirrB5Nur5UlV2/T81zd6W09YLlhoiIyEHJZddO6Tcnqc9Vctxl10RERGQRUi9OZrkhIiIiu8JyQ0RERHaF5YaIiIjsCssNERER2RWWGyIiIrIrLDdERERkV1huiIiIyK6w3BAREZFdYbkhIiIiu8JyQ0RERHaF5YaIiIjsCssNERER2RWWGyIiIrIr5r3HuQ2ovQ27RqOROAkRERE1Vu3P7dqf4w1xuHJTVFQEAAgJCZE4CREREZmqqKgIarW6wX0EsTEVyI7o9XpcvHgRHh4eEATBrO+t0WgQEhKCzMxMeHp6mvW9yXT8flgXfj+sD78n1oXfj4aJooiioiIEBwdDJmt4VY3DjdzIZDK0aNHCop/D09OTfzCtCL8f1oXfD+vD74l14ffjxm42YlOLC4qJiIjIrrDcEBERkV1huTEjpVKJ2bNnQ6lUSh2FwO+HteH3w/rwe2Jd+P0wH4dbUExERET2jSM3REREZFdYboiIiMiusNwQERGRXWG5ISIiIrvCcmMmS5cuRWhoKFQqFWJiYrBv3z6pIzmspKQk9OjRAx4eHvD398ewYcNw+vRpqWNRjf/7v/+DIAiYPHmy1FEcVlZWFp544gn4+vrCxcUFnTt3xoEDB6SO5ZB0Oh1mzpyJsLAwuLi4IDw8HG+++Waj7p9EN8ZyYwYrV65EYmIiZs+ejUOHDiEqKgoDBw5Ebm6u1NEc0o4dOzBhwgTs3bsXW7ZsQWVlJQYMGICSkhKpozm8/fv346OPPkKXLl2kjuKwrl69ij59+sDZ2RkbN27EiRMn8N5778Hb21vqaA7p7bffxrJly7BkyRKcPHkSb7/9NhYsWIAPPvhA6mg2jaeCm0FMTAx69OiBJUuWAKi+f1VISAhefPFFTJ8+XeJ0dPnyZfj7+2PHjh24++67pY7jsIqLi3HHHXfgww8/xFtvvYWuXbti8eLFUsdyONOnT8fu3bvxxx9/SB2FAAwdOhQBAQFYsWKFYdvw4cPh4uKCr776SsJkto0jN7epoqICBw8eRP/+/Q3bZDIZ+vfvjz179kiYjGoVFhYCAHx8fCRO4tgmTJiAIUOGGP1doab3008/oXv37nj00Ufh7++P6OhofPLJJ1LHcli9e/fG1q1bkZKSAgA4cuQIdu3ahbi4OImT2TaHu3GmueXl5UGn0yEgIMBoe0BAAE6dOiVRKqql1+sxefJk9OnTB506dZI6jsP67rvvcOjQIezfv1/qKA4vNTUVy5YtQ2JiIl599VXs378fkyZNgkKhQEJCgtTxHM706dOh0WjQrl07yOVy6HQ6zJs3D6NHj5Y6mk1juSG7NmHCBPz999/YtWuX1FEcVmZmJl566SVs2bIFKpVK6jgOT6/Xo3v37pg/fz4AIDo6Gn///TeWL1/OciOBVatW4euvv8Y333yDjh07Ijk5GZMnT0ZwcDC/H7eB5eY2+fn5QS6XIycnx2h7Tk4OAgMDJUpFADBx4kT88ssv2LlzJ1q0aCF1HId18OBB5Obm4o477jBs0+l02LlzJ5YsWQKtVgu5XC5hQscSFBSEDh06GG1r3749vv/+e4kSObaXX34Z06dPx+OPPw4A6Ny5M86fP4+kpCSWm9vANTe3SaFQoFu3bti6dathm16vx9atWxEbGythMscliiImTpyIH3/8Edu2bUNYWJjUkRxav379cOzYMSQnJxse3bt3x+jRo5GcnMxi08T69OlT59IIKSkpaNWqlUSJHFtpaSlkMuMfxXK5HHq9XqJE9oEjN2aQmJiIhIQEdO/eHT179sTixYtRUlKCcePGSR3NIU2YMAHffPMN1q1bBw8PD2RnZwMA1Go1XFxcJE7neDw8POqsd3Jzc4Ovry/XQUlgypQp6N27N+bPn4/HHnsM+/btw8cff4yPP/5Y6mgOKT4+HvPmzUPLli3RsWNHHD58GAsXLsT48eOljmbTeCq4mSxZsgTvvPMOsrOz0bVrV7z//vuIiYmROpZDEgSh3u2ff/45xo4d27RhqF733HMPTwWX0C+//IIZM2bgzJkzCAsLQ2JiIp555hmpYzmkoqIizJw5Ez/++CNyc3MRHByMkSNHYtasWVAoFFLHs1ksN0RERGRXuOaGiIiI7ArLDREREdkVlhsiIiKyKyw3REREZFdYboiIiMiusNwQERGRXWG5ISIiIrvCckNEhOqLP65du1bqGERkBiw3RCS5sWPHQhCEOo9BgwZJHY2IbBDvLUVEVmHQoEH4/PPPjbYplUqJ0hCRLePIDRFZBaVSicDAQKOHt7c3gOopo2XLliEuLg4uLi5o3bo11qxZY3T8sWPHcN9998HFxQW+vr549tlnUVxcbLTPZ599ho4dO0KpVCIoKAgTJ040ej0vLw8PPfQQXF1d0aZNG/z000+W/aKJyCJYbojIJsycORPDhw/HkSNHMHr0aDz++OM4efIkAKCkpAQDBw6Et7c39u/fj9WrV+O3334zKi/Lli3DhAkT8Oyzz+LYsWP46aefEBERYfQ55syZg8ceewxHjx7F4MGDMXr0aOTn5zfp10lEZiASEUksISFBlMvlopubm9Fj3rx5oiiKIgDxueeeMzomJiZGfP7550VRFMWPP/5Y9Pb2FouLiw2vr1+/XpTJZGJ2drYoiqIYHBwsvvbaazfMAEB8/fXXDc+Li4tFAOLGjRvN9nUSUdPgmhsisgr33nsvli1bZrTNx8fH8HFsbKzRa7GxsUhOTgYAnDx5ElFRUXBzczO83qdPH+j1epw+fRqCIODixYvo169fgxm6dOli+NjNzQ2enp7Izc291S+JiCTCckNEVsHNza3ONJG5uLi4NGo/Z2dno+eCIECv11siEhFZENfcEJFN2Lt3b53n7du3BwC0b98eR44cQUlJieH13bt3QyaTITIyEh4eHggNDcXWrVubNDMRSYMjN0RkFbRaLbKzs422OTk5wc/PDwCwevVqdO/eHXfeeSe+/vpr7Nu3DytWrAAAjB49GrNnz0ZCQgLeeOMNXL58GS+++CKefPJJBAQEAADeeOMNPPfcc/D390dcXByKioqwe/duvPjii037hRKRxbHcEJFV2LRpE4KCgoy2RUZG4tSpUwCqz2T67rvv8MILLyAoKAjffvstOnToAABwdXXF5s2b8dJLL6FHjx5wdXXF8OHDsXDhQsN7JSQkoLy8HIsWLcK0adPg5+eHRx55pOm+QCJqMoIoiqLUIYiIGiIIAn788UcMGzZM6ihEZAO45oaIiIjsCssNERER2RWuuSEiq8fZcyIyBUduiIiIyK6w3BAREZFdYbkhIiIiu8JyQ0RERHaF5YaIiIjsCssNERER2RWWGyIiIrIrLDdERERkV1huiIiIyK78Pw7J38FObGr6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Training Loss vs Epoch\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Training Loss\")\n",
    "plt.plot(range(epochs), losses)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1622a99c",
   "metadata": {},
   "source": [
    "#### Calculating Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5fd4c189",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9028\n"
     ]
    }
   ],
   "source": [
    "# Load the test data\n",
    "test_dataset = CustomMNISTDataset('./output/testing/')\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# Set model to eval mode\n",
    "model.eval()\n",
    "\n",
    "# Loop over dataset to compute accuracy\n",
    "correct = 0\n",
    "for batch_idx, (data, targets) in enumerate(test_dataloader):\n",
    "    outputs = model(data)\n",
    "    pred = outputs.argmax(dim=1, keepdim=True) \n",
    "    correct += pred.eq(targets.argmax(dim=2).view_as(pred)).sum().item()\n",
    "accuracy = correct / test_dataset.__len__()\n",
    "print(f\"Accuracy: {accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b97e779",
   "metadata": {},
   "source": [
    "#### Encapsulating Entire Training and Validation into one Function\n",
    "#### Trainer Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ba11be43",
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainer(config, epochs=10, data_dir=None):\n",
    "    # Loading Dataset\n",
    "    train_dataset = CustomMNISTDataset(data_dir+'/training/')\n",
    "    train_dataloader = DataLoader(train_dataset, batch_size=config[\"batch_size\"], shuffle=True)\n",
    "\n",
    "    # Initialize Model\n",
    "    model = NeuralNet(config[\"hidden_layer_1\"], config[\"hidden_layer_2\"])\n",
    "    # Initialize Optimizer\n",
    "    optimizer = optim.Adadelta(model.parameters(), lr=config[\"lr\"])\n",
    "    # Loss function\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    \n",
    "    checkpoint = session.get_checkpoint()\n",
    "\n",
    "    if checkpoint:\n",
    "        checkpoint_state = checkpoint.to_dict()\n",
    "        start_epoch = checkpoint_state[\"epoch\"]\n",
    "        net.load_state_dict(checkpoint_state[\"net_state_dict\"])\n",
    "        optimizer.load_state_dict(checkpoint_state[\"optimizer_state_dict\"])\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "\n",
    "    # set model to train mode\n",
    "    losses = []\n",
    "    model.train()\n",
    "    for epoch in range(epochs):\n",
    "        batch_losses = []    \n",
    "        for batch_idx, (data, targets) in enumerate(train_dataloader):\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(data)\n",
    "            loss = loss_fn(outputs, targets.squeeze(1).float())\n",
    "            loss.backward() # Backpropagation of loss\n",
    "            optimizer.step() # Updating Weights based on Gradients\n",
    "            batch_losses.append(loss.item())\n",
    "        final_epoch_loss = torch.mean(torch.Tensor(batch_losses))\n",
    "        losses.append(final_epoch_loss.item())\n",
    "        \n",
    "    # Load the test data\n",
    "    test_dataset = CustomMNISTDataset(data_dir+'/testing/')\n",
    "    test_dataloader = DataLoader(test_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "    # Set model to eval mode\n",
    "    model.eval()\n",
    "\n",
    "    # Loop over dataset to compute accuracy\n",
    "    correct = 0\n",
    "    for batch_idx, (data, targets) in enumerate(test_dataloader):\n",
    "        outputs = model(data)\n",
    "        pred = outputs.argmax(dim=1, keepdim=True) \n",
    "        correct += pred.eq(targets.argmax(dim=2).view_as(pred)).sum().item()\n",
    "    accuracy = correct / test_dataset.__len__()\n",
    "    session.report({\"loss\": sum(losses) / len(losses), \"accuracy\": accuracy})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47ebfdd",
   "metadata": {},
   "source": [
    "#### Setting Up Ray Tune Configs and Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "64cc1f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from ray import tune\n",
    "from ray.air import Checkpoint, session\n",
    "from ray.tune.schedulers import ASHAScheduler\n",
    "\n",
    "max_num_epochs = 10\n",
    "num_samples = 10\n",
    "\n",
    "config = {\n",
    "\t\"hidden_layer_1\": tune.choice([512, 256]),\n",
    "\t\"hidden_layer_2\": tune.choice([128, 64]),\n",
    "\t\"lr\": tune.loguniform(1e-4, 1e-1),\n",
    "\t\"batch_size\": tune.choice([32, 48, 64])}\n",
    "\n",
    "scheduler = ASHAScheduler(\n",
    "        metric=\"loss\",\n",
    "        mode=\"min\",\n",
    "        max_t=max_num_epochs,\n",
    "        grace_period=1,\n",
    "        reduction_factor=2,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b67b9997",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-18 21:39:56,322\tINFO worker.py:1642 -- Started a local Ray instance.\n",
      "2023-10-18 21:39:57,151\tINFO tune.py:228 -- Initializing Ray automatically. For cluster usage or custom Ray initialization, call `ray.init(...)` before `tune.run(...)`.\n",
      "2023-10-18 21:39:57,153\tINFO tune.py:645 -- [output] This uses the legacy output and progress reporter, as Jupyter notebooks are not supported by the new engine, yet. For more information, please see https://github.com/ray-project/ray/issues/36949\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-10-18 21:42:44</td></tr>\n",
       "<tr><td>Running for: </td><td>00:02:46.98        </td></tr>\n",
       "<tr><td>Memory:      </td><td>7.6/8.0 GiB        </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using AsyncHyperBand: num_stopped=3<br>Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: None | Iter 1.000: -1.4415028870105742<br>Logical resource usage: 1.0/8 CPUs, 0/0 GPUs\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "<div class=\"messages\">\n",
       "  <h3>Messages</h3>\n",
       "  : ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.\n",
       "  \n",
       "  \n",
       "</div>\n",
       "<style>\n",
       ".messages {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  padding-left: 1em;\n",
       "  overflow-y: auto;\n",
       "}\n",
       ".messages h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n",
       "\n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name         </th><th>status    </th><th>loc            </th><th style=\"text-align: right;\">  batch_size</th><th style=\"text-align: right;\">  hidden_layer_1</th><th style=\"text-align: right;\">  hidden_layer_2</th><th style=\"text-align: right;\">         lr</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">    loss</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>trainer_8e1f6_00000</td><td>TERMINATED</td><td>127.0.0.1:25814</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.0131781  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         97.118 </td><td style=\"text-align: right;\">0.538548</td><td style=\"text-align: right;\">    0.9109</td></tr>\n",
       "<tr><td>trainer_8e1f6_00001</td><td>TERMINATED</td><td>127.0.0.1:25815</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.00214634 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         91.3011</td><td style=\"text-align: right;\">1.47806 </td><td style=\"text-align: right;\">    0.8061</td></tr>\n",
       "<tr><td>trainer_8e1f6_00002</td><td>TERMINATED</td><td>127.0.0.1:25816</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.00129984 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        103.354 </td><td style=\"text-align: right;\">1.40495 </td><td style=\"text-align: right;\">    0.8321</td></tr>\n",
       "<tr><td>trainer_8e1f6_00003</td><td>TERMINATED</td><td>127.0.0.1:25817</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             256</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.000242174</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         90.0152</td><td style=\"text-align: right;\">2.16802 </td><td style=\"text-align: right;\">    0.6958</td></tr>\n",
       "<tr><td>trainer_8e1f6_00004</td><td>TERMINATED</td><td>127.0.0.1:25818</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.0013529  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         97.2941</td><td style=\"text-align: right;\">1.5946  </td><td style=\"text-align: right;\">    0.8054</td></tr>\n",
       "<tr><td>trainer_8e1f6_00005</td><td>TERMINATED</td><td>127.0.0.1:25819</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.012783   </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        103.176 </td><td style=\"text-align: right;\">0.482466</td><td style=\"text-align: right;\">    0.9141</td></tr>\n",
       "<tr><td>trainer_8e1f6_00006</td><td>TERMINATED</td><td>127.0.0.1:25820</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.000110933</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         99.5073</td><td style=\"text-align: right;\">2.24629 </td><td style=\"text-align: right;\">    0.5104</td></tr>\n",
       "<tr><td>trainer_8e1f6_00007</td><td>TERMINATED</td><td>127.0.0.1:25821</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.00190065 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         97.4514</td><td style=\"text-align: right;\">1.34412 </td><td style=\"text-align: right;\">    0.8383</td></tr>\n",
       "<tr><td>trainer_8e1f6_00008</td><td>TERMINATED</td><td>127.0.0.1:25817</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">             256</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.000187373</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         40.9069</td><td style=\"text-align: right;\">2.25873 </td><td style=\"text-align: right;\">    0.415 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00009</td><td>TERMINATED</td><td>127.0.0.1:25815</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.0127463  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         71.4595</td><td style=\"text-align: right;\">0.468081</td><td style=\"text-align: right;\">    0.917 </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name         </th><th style=\"text-align: right;\">  accuracy</th><th style=\"text-align: right;\">    loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>trainer_8e1f6_00000</td><td style=\"text-align: right;\">    0.9109</td><td style=\"text-align: right;\">0.538548</td></tr>\n",
       "<tr><td>trainer_8e1f6_00001</td><td style=\"text-align: right;\">    0.8061</td><td style=\"text-align: right;\">1.47806 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00002</td><td style=\"text-align: right;\">    0.8321</td><td style=\"text-align: right;\">1.40495 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00003</td><td style=\"text-align: right;\">    0.6958</td><td style=\"text-align: right;\">2.16802 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00004</td><td style=\"text-align: right;\">    0.8054</td><td style=\"text-align: right;\">1.5946  </td></tr>\n",
       "<tr><td>trainer_8e1f6_00005</td><td style=\"text-align: right;\">    0.9141</td><td style=\"text-align: right;\">0.482466</td></tr>\n",
       "<tr><td>trainer_8e1f6_00006</td><td style=\"text-align: right;\">    0.5104</td><td style=\"text-align: right;\">2.24629 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00007</td><td style=\"text-align: right;\">    0.8383</td><td style=\"text-align: right;\">1.34412 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00008</td><td style=\"text-align: right;\">    0.415 </td><td style=\"text-align: right;\">2.25873 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00009</td><td style=\"text-align: right;\">    0.917 </td><td style=\"text-align: right;\">0.468081</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-18 21:42:44,171\tINFO tune.py:1143 -- Total run time: 167.02 seconds (166.98 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best trial config: {'hidden_layer_1': 512, 'hidden_layer_2': 128, 'lr': 0.012746297610971511, 'batch_size': 32}\n",
      "Best trial final validation loss:  0.46808149814605715\n",
      "Best trial final validation loss:  0.917\n"
     ]
    }
   ],
   "source": [
    "result = tune.run(\n",
    "        partial(trainer,epochs=5, data_dir='/Users/atifadib/Deep_Learning_with_PyTorch/chapter3/output'),\n",
    "        resources_per_trial={\"cpu\": 1, \"gpus_per_trial\": 0},\n",
    "        config=config,\n",
    "        num_samples=num_samples,\n",
    "        scheduler=scheduler,\n",
    "    )\n",
    "\n",
    "best_trial = result.get_best_trial(\"loss\", \"min\", \"last\")\n",
    "print(f\"Best trial config: {best_trial.config}\")\n",
    "print(f\"Best trial final validation loss:  {best_trial.last_result['loss']}\")\n",
    "print(f\"Best trial final validation loss:  {best_trial.last_result['accuracy']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3798fe4d",
   "metadata": {},
   "source": [
    "#### Saving and Reloading a Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4f1bac40",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'model.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2f62cd20",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_new = NeuralNet(512, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "024c5690",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_new.load_state_dict(torch.load('model.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eebdeba2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
